Spaces:
Running
Running
| { | |
| "reference_repo": "RLE-Bench/kinex-v0.10.0-benchmark", | |
| "reference_delivery_time": "2026-10-09T17:52:51.393664+00:00", | |
| "reference_revisions": { | |
| "dataset": "07dd924831cf10515ccda0a05d4a8ce169df7965", | |
| "space": "83483dadd2a7140a4f23bd247ff91183444b1da5" | |
| }, | |
| "families": [ | |
| { | |
| "family": "task01", | |
| "tasks": 10, | |
| "codex_successes": 2, | |
| "kinex_successes": 1, | |
| "codex_usage": { | |
| "input_tokens": 63285354, | |
| "cached_input_tokens": 62436736, | |
| "uncached_input_tokens": 848618, | |
| "output_tokens": 204733, | |
| "reasoning_output_tokens": 78505, | |
| "response_count": 1112, | |
| "cost_usd": null, | |
| "attempts": 10, | |
| "reported_attempts": { | |
| "input_tokens": 10, | |
| "cached_input_tokens": 10, | |
| "uncached_input_tokens": 10, | |
| "output_tokens": 10, | |
| "reasoning_output_tokens": 10, | |
| "response_count": 10, | |
| "cost_usd": 0 | |
| }, | |
| "audit_complete_attempts": 10, | |
| "cache_hit_rate": 0.9865906098905601, | |
| "tokens_per_task": { | |
| "input_tokens": 6328535.4, | |
| "cached_input_tokens": 6243673.6, | |
| "uncached_input_tokens": 84861.8, | |
| "output_tokens": 20473.3, | |
| "reasoning_output_tokens": 7850.5 | |
| } | |
| }, | |
| "kinex_usage": { | |
| "input_tokens": 120363624, | |
| "cached_input_tokens": 118975104, | |
| "uncached_input_tokens": 1388520, | |
| "output_tokens": 247504, | |
| "reasoning_output_tokens": 98713, | |
| "response_count": 2003, | |
| "cost_usd": null, | |
| "attempts": 10, | |
| "reported_attempts": { | |
| "input_tokens": 10, | |
| "cached_input_tokens": 10, | |
| "uncached_input_tokens": 10, | |
| "output_tokens": 10, | |
| "reasoning_output_tokens": 10, | |
| "response_count": 10, | |
| "cost_usd": 0 | |
| }, | |
| "audit_complete_attempts": 10, | |
| "cache_hit_rate": 0.9884639565189562, | |
| "tokens_per_task": { | |
| "input_tokens": 12036362.4, | |
| "cached_input_tokens": 11897510.4, | |
| "uncached_input_tokens": 138852.0, | |
| "output_tokens": 24750.4, | |
| "reasoning_output_tokens": 9871.3 | |
| } | |
| }, | |
| "codex_wall_time": { | |
| "reported_attempts": 10, | |
| "sum_s": 11720.851507, | |
| "mean_s": 1172.0851507, | |
| "median_s": 1239.5881319999999 | |
| }, | |
| "kinex_wall_time": { | |
| "reported_attempts": 10, | |
| "sum_s": 21798.805297, | |
| "mean_s": 2179.8805297, | |
| "median_s": 2131.8586855000003 | |
| } | |
| }, | |
| { | |
| "family": "task02", | |
| "tasks": 10, | |
| "codex_successes": 9, | |
| "kinex_successes": 10, | |
| "codex_usage": { | |
| "input_tokens": 12586499, | |
| "cached_input_tokens": 12106752, | |
| "uncached_input_tokens": 479747, | |
| "output_tokens": 93969, | |
| "reasoning_output_tokens": 34987, | |
| "response_count": 344, | |
| "cost_usd": null, | |
| "attempts": 10, | |
| "reported_attempts": { | |
| "input_tokens": 10, | |
| "cached_input_tokens": 10, | |
| "uncached_input_tokens": 10, | |
| "output_tokens": 10, | |
| "reasoning_output_tokens": 10, | |
| "response_count": 10, | |
| "cost_usd": 0 | |
| }, | |
| "audit_complete_attempts": 10, | |
| "cache_hit_rate": 0.9618839996729829, | |
| "tokens_per_task": { | |
| "input_tokens": 1258649.9, | |
| "cached_input_tokens": 1210675.2, | |
| "uncached_input_tokens": 47974.7, | |
| "output_tokens": 9396.9, | |
| "reasoning_output_tokens": 3498.7 | |
| } | |
| }, | |
| "kinex_usage": { | |
| "input_tokens": 17966387, | |
| "cached_input_tokens": 17024128, | |
| "uncached_input_tokens": 942259, | |
| "output_tokens": 83439, | |
| "reasoning_output_tokens": 26298, | |
| "response_count": 599, | |
| "cost_usd": null, | |
| "attempts": 10, | |
| "reported_attempts": { | |
| "input_tokens": 10, | |
| "cached_input_tokens": 10, | |
| "uncached_input_tokens": 10, | |
| "output_tokens": 10, | |
| "reasoning_output_tokens": 10, | |
| "response_count": 10, | |
| "cost_usd": 0 | |
| }, | |
| "audit_complete_attempts": 10, | |
| "cache_hit_rate": 0.947554341337521, | |
| "tokens_per_task": { | |
| "input_tokens": 1796638.7, | |
| "cached_input_tokens": 1702412.8, | |
| "uncached_input_tokens": 94225.9, | |
| "output_tokens": 8343.9, | |
| "reasoning_output_tokens": 2629.8 | |
| } | |
| }, | |
| "codex_wall_time": { | |
| "reported_attempts": 10, | |
| "sum_s": 3681.418654, | |
| "mean_s": 368.14186540000003, | |
| "median_s": 266.88804849999997 | |
| }, | |
| "kinex_wall_time": { | |
| "reported_attempts": 10, | |
| "sum_s": 4811.317263, | |
| "mean_s": 481.1317263, | |
| "median_s": 409.560245 | |
| } | |
| } | |
| ], | |
| "tasks": [ | |
| { | |
| "task_key": "task01/01", | |
| "instruction": "Place the specified condiments from the counter on the top shelf of the fridge. Move any existing top-shelf items to other shelves. Release the items and move the gripper clear of the stored items.", | |
| "instruction_policy": "modified", | |
| "codex_success": false, | |
| "kinex_success": false, | |
| "codex_steps": 5496, | |
| "kinex_steps": 3704, | |
| "codex_usage": { | |
| "accounting": "reported-responses", | |
| "audit_complete": true, | |
| "cache_hit_rate": 0.9875774826559943, | |
| "cache_reported_input_tokens": 7077229, | |
| "cache_write_input_tokens": 0, | |
| "cache_write_reported_input_tokens": 7077229, | |
| "cached_input_tokens": 6989312, | |
| "completed_turns": 1, | |
| "cost_usd": null, | |
| "failed_turns": 0, | |
| "input_tokens": 7077229, | |
| "known_cache_write_input_tokens": 0, | |
| "known_cached_input_tokens": 6989312, | |
| "known_input_tokens": 7077229, | |
| "known_output_tokens": 20496, | |
| "known_reasoning_output_tokens": 5873, | |
| "output_tokens": 20496, | |
| "reasoning_output_tokens": 5873, | |
| "reasoning_reported_output_tokens": 20496, | |
| "reported_responses": { | |
| "cache_reported_input_tokens": 140, | |
| "cache_write_input_tokens": 140, | |
| "cache_write_reported_input_tokens": 140, | |
| "cached_input_tokens": 140, | |
| "input_tokens": 140, | |
| "output_tokens": 140, | |
| "reasoning_output_tokens": 140, | |
| "reasoning_reported_output_tokens": 140 | |
| }, | |
| "response_count": 140, | |
| "response_ids_complete": true, | |
| "schema": "rlebench/token-usage/1", | |
| "source": "Codex token_usage_record per response", | |
| "uncached_input_tokens": 87917, | |
| "unidentified_usage_records": 0 | |
| }, | |
| "kinex_usage": { | |
| "input_tokens": 9503123, | |
| "output_tokens": 23563, | |
| "cached_input_tokens": 9397760, | |
| "cache_write_input_tokens": 0, | |
| "reasoning_output_tokens": 8032, | |
| "cache_reported_input_tokens": 9503123, | |
| "cache_write_reported_input_tokens": 9503123, | |
| "reasoning_reported_output_tokens": 23563, | |
| "schema": "rlebench/token-usage/1", | |
| "source": "Provider terminal response usage", | |
| "response_count": 191, | |
| "reported_responses": { | |
| "input_tokens": 191, | |
| "output_tokens": 191, | |
| "cached_input_tokens": 191, | |
| "cache_write_input_tokens": 191, | |
| "reasoning_output_tokens": 191, | |
| "cache_reported_input_tokens": 191, | |
| "cache_write_reported_input_tokens": 191, | |
| "reasoning_reported_output_tokens": 191 | |
| }, | |
| "known_cached_input_tokens": 9397760, | |
| "known_input_tokens": 9503123, | |
| "known_output_tokens": 23563, | |
| "known_cache_write_input_tokens": 0, | |
| "known_reasoning_output_tokens": 8032, | |
| "cache_hit_rate": 0.9889128026649766, | |
| "uncached_input_tokens": 105363, | |
| "cost_usd": null, | |
| "response_ids_complete": true, | |
| "usage_source": "provider-raw", | |
| "request_attempts": 192, | |
| "rejected_attempts": 0, | |
| "unresolved_attempts": [], | |
| "audit_complete": true, | |
| "accounting": "reported-responses" | |
| }, | |
| "codex_wall_time_s": 1267.271401, | |
| "kinex_wall_time_s": 1805.038681, | |
| "kinex_version": "0.10.3", | |
| "kinex_revision": "a1eaece0bd01a25177a43c2091c10b213b416aa5", | |
| "same_roboenv_source": true | |
| }, | |
| { | |
| "task_key": "task01/02", | |
| "instruction": "Remove the specified fruit from the bowl and place it on the small plate. Then place the bowl with only the specified meat in the microwave, close the door, and press the start button to microwave the meat. Release the bowl and move the gripper clear of it.", | |
| "instruction_policy": "modified", | |
| "codex_success": false, | |
| "kinex_success": false, | |
| "codex_steps": 5903, | |
| "kinex_steps": 5048, | |
| "codex_usage": { | |
| "accounting": "reported-responses", | |
| "audit_complete": true, | |
| "cache_hit_rate": 0.9810887797632594, | |
| "cache_reported_input_tokens": 4438106, | |
| "cache_write_input_tokens": 0, | |
| "cache_write_reported_input_tokens": 4438106, | |
| "cached_input_tokens": 4354176, | |
| "completed_turns": 1, | |
| "cost_usd": null, | |
| "failed_turns": 0, | |
| "input_tokens": 4438106, | |
| "known_cache_write_input_tokens": 0, | |
| "known_cached_input_tokens": 4354176, | |
| "known_input_tokens": 4438106, | |
| "known_output_tokens": 22048, | |
| "known_reasoning_output_tokens": 9003, | |
| "output_tokens": 22048, | |
| "reasoning_output_tokens": 9003, | |
| "reasoning_reported_output_tokens": 22048, | |
| "reported_responses": { | |
| "cache_reported_input_tokens": 86, | |
| "cache_write_input_tokens": 86, | |
| "cache_write_reported_input_tokens": 86, | |
| "cached_input_tokens": 86, | |
| "input_tokens": 86, | |
| "output_tokens": 86, | |
| "reasoning_output_tokens": 86, | |
| "reasoning_reported_output_tokens": 86 | |
| }, | |
| "response_count": 86, | |
| "response_ids_complete": true, | |
| "schema": "rlebench/token-usage/1", | |
| "source": "Codex token_usage_record per response", | |
| "uncached_input_tokens": 83930, | |
| "unidentified_usage_records": 0 | |
| }, | |
| "kinex_usage": { | |
| "input_tokens": 13113774, | |
| "output_tokens": 31080, | |
| "cached_input_tokens": 12981632, | |
| "cache_write_input_tokens": 0, | |
| "reasoning_output_tokens": 13344, | |
| "cache_reported_input_tokens": 13113774, | |
| "cache_write_reported_input_tokens": 13113774, | |
| "reasoning_reported_output_tokens": 31080, | |
| "schema": "rlebench/token-usage/1", | |
| "source": "Provider terminal response usage", | |
| "response_count": 214, | |
| "reported_responses": { | |
| "input_tokens": 214, | |
| "output_tokens": 214, | |
| "cached_input_tokens": 214, | |
| "cache_write_input_tokens": 214, | |
| "reasoning_output_tokens": 214, | |
| "cache_reported_input_tokens": 214, | |
| "cache_write_reported_input_tokens": 214, | |
| "reasoning_reported_output_tokens": 214 | |
| }, | |
| "known_cached_input_tokens": 12981632, | |
| "known_input_tokens": 13113774, | |
| "known_output_tokens": 31080, | |
| "known_cache_write_input_tokens": 0, | |
| "known_reasoning_output_tokens": 13344, | |
| "cache_hit_rate": 0.9899234194519442, | |
| "uncached_input_tokens": 132142, | |
| "cost_usd": null, | |
| "response_ids_complete": true, | |
| "usage_source": "provider-raw", | |
| "request_attempts": 215, | |
| "rejected_attempts": 0, | |
| "unresolved_attempts": [], | |
| "audit_complete": true, | |
| "accounting": "reported-responses" | |
| }, | |
| "codex_wall_time_s": 1211.904863, | |
| "kinex_wall_time_s": 2458.67869, | |
| "kinex_version": "0.10.3", | |
| "kinex_revision": "a1eaece0bd01a25177a43c2091c10b213b416aa5", | |
| "same_roboenv_source": true | |
| }, | |
| { | |
| "task_key": "task01/03", | |
| "instruction": "Place two dumplings into each of the tupperware containers and then place both containers on a shelf in the fridge. Release the dumplings and containers and move the gripper clear of them.", | |
| "instruction_policy": "modified", | |
| "codex_success": false, | |
| "kinex_success": false, | |
| "codex_steps": 5053, | |
| "kinex_steps": 2809, | |
| "codex_usage": { | |
| "accounting": "reported-responses", | |
| "audit_complete": true, | |
| "cache_hit_rate": 0.9899699443798293, | |
| "cache_reported_input_tokens": 11115392, | |
| "cache_write_input_tokens": 0, | |
| "cache_write_reported_input_tokens": 11115392, | |
| "cached_input_tokens": 11003904, | |
| "completed_turns": 1, | |
| "cost_usd": null, | |
| "failed_turns": 0, | |
| "input_tokens": 11115392, | |
| "known_cache_write_input_tokens": 0, | |
| "known_cached_input_tokens": 11003904, | |
| "known_input_tokens": 11115392, | |
| "known_output_tokens": 29598, | |
| "known_reasoning_output_tokens": 14166, | |
| "output_tokens": 29598, | |
| "reasoning_output_tokens": 14166, | |
| "reasoning_reported_output_tokens": 29598, | |
| "reported_responses": { | |
| "cache_reported_input_tokens": 171, | |
| "cache_write_input_tokens": 171, | |
| "cache_write_reported_input_tokens": 171, | |
| "cached_input_tokens": 171, | |
| "input_tokens": 171, | |
| "output_tokens": 171, | |
| "reasoning_output_tokens": 171, | |
| "reasoning_reported_output_tokens": 171 | |
| }, | |
| "response_count": 171, | |
| "response_ids_complete": true, | |
| "schema": "rlebench/token-usage/1", | |
| "source": "Codex token_usage_record per response", | |
| "uncached_input_tokens": 111488, | |
| "unidentified_usage_records": 0 | |
| }, | |
| "kinex_usage": { | |
| "input_tokens": 6179963, | |
| "output_tokens": 17710, | |
| "cached_input_tokens": 6033792, | |
| "cache_write_input_tokens": 0, | |
| "reasoning_output_tokens": 5948, | |
| "cache_reported_input_tokens": 6179963, | |
| "cache_write_reported_input_tokens": 6179963, | |
| "reasoning_reported_output_tokens": 17710, | |
| "schema": "rlebench/token-usage/1", | |
| "source": "Provider terminal response usage", | |
| "response_count": 135, | |
| "reported_responses": { | |
| "input_tokens": 135, | |
| "output_tokens": 135, | |
| "cached_input_tokens": 135, | |
| "cache_write_input_tokens": 135, | |
| "reasoning_output_tokens": 135, | |
| "cache_reported_input_tokens": 135, | |
| "cache_write_reported_input_tokens": 135, | |
| "reasoning_reported_output_tokens": 135 | |
| }, | |
| "known_cached_input_tokens": 6033792, | |
| "known_input_tokens": 6179963, | |
| "known_output_tokens": 17710, | |
| "known_cache_write_input_tokens": 0, | |
| "known_reasoning_output_tokens": 5948, | |
| "cache_hit_rate": 0.9763475930195699, | |
| "uncached_input_tokens": 146171, | |
| "cost_usd": null, | |
| "response_ids_complete": true, | |
| "usage_source": "provider-raw", | |
| "request_attempts": 135, | |
| "rejected_attempts": 0, | |
| "unresolved_attempts": [], | |
| "audit_complete": true, | |
| "accounting": "reported-responses" | |
| }, | |
| "codex_wall_time_s": 1749.701956, | |
| "kinex_wall_time_s": 1444.988461, | |
| "kinex_version": "0.10.3", | |
| "kinex_revision": "a1eaece0bd01a25177a43c2091c10b213b416aa5", | |
| "same_roboenv_source": true | |
| }, | |
| { | |
| "task_key": "task01/04", | |
| "instruction": "Gather the specified vegetables from the fridge and place them on a tray on the dining counter. Then gather the specified meats from the fridge and place them on the other tray. Release the food and move the gripper clear of the food and both trays.", | |
| "instruction_policy": "modified", | |
| "codex_success": false, | |
| "kinex_success": false, | |
| "codex_steps": 5435, | |
| "kinex_steps": 5449, | |
| "codex_usage": { | |
| "accounting": "reported-responses", | |
| "audit_complete": true, | |
| "cache_hit_rate": 0.9899239913597869, | |
| "cache_reported_input_tokens": 11714063, | |
| "cache_write_input_tokens": 0, | |
| "cache_write_reported_input_tokens": 11714063, | |
| "cached_input_tokens": 11596032, | |
| "completed_turns": 1, | |
| "cost_usd": null, | |
| "failed_turns": 0, | |
| "input_tokens": 11714063, | |
| "known_cache_write_input_tokens": 0, | |
| "known_cached_input_tokens": 11596032, | |
| "known_input_tokens": 11714063, | |
| "known_output_tokens": 32111, | |
| "known_reasoning_output_tokens": 11814, | |
| "output_tokens": 32111, | |
| "reasoning_output_tokens": 11814, | |
| "reasoning_reported_output_tokens": 32111, | |
| "reported_responses": { | |
| "cache_reported_input_tokens": 190, | |
| "cache_write_input_tokens": 190, | |
| "cache_write_reported_input_tokens": 190, | |
| "cached_input_tokens": 190, | |
| "input_tokens": 190, | |
| "output_tokens": 190, | |
| "reasoning_output_tokens": 190, | |
| "reasoning_reported_output_tokens": 190 | |
| }, | |
| "response_count": 190, | |
| "response_ids_complete": true, | |
| "schema": "rlebench/token-usage/1", | |
| "source": "Codex token_usage_record per response", | |
| "uncached_input_tokens": 118031, | |
| "unidentified_usage_records": 0 | |
| }, | |
| "kinex_usage": { | |
| "input_tokens": 18456926, | |
| "output_tokens": 44833, | |
| "cached_input_tokens": 18169984, | |
| "cache_write_input_tokens": 0, | |
| "reasoning_output_tokens": 22099, | |
| "cache_reported_input_tokens": 18456926, | |
| "cache_write_reported_input_tokens": 18456926, | |
| "reasoning_reported_output_tokens": 44833, | |
| "schema": "rlebench/token-usage/1", | |
| "source": "Provider terminal response usage", | |
| "response_count": 236, | |
| "reported_responses": { | |
| "input_tokens": 236, | |
| "output_tokens": 236, | |
| "cached_input_tokens": 236, | |
| "cache_write_input_tokens": 236, | |
| "reasoning_output_tokens": 236, | |
| "cache_reported_input_tokens": 236, | |
| "cache_write_reported_input_tokens": 236, | |
| "reasoning_reported_output_tokens": 236 | |
| }, | |
| "known_cached_input_tokens": 18169984, | |
| "known_input_tokens": 18456926, | |
| "known_output_tokens": 44833, | |
| "known_cache_write_input_tokens": 0, | |
| "known_reasoning_output_tokens": 22099, | |
| "cache_hit_rate": 0.9844534241509122, | |
| "uncached_input_tokens": 286942, | |
| "cost_usd": null, | |
| "response_ids_complete": true, | |
| "usage_source": "provider-raw", | |
| "request_attempts": 237, | |
| "rejected_attempts": 0, | |
| "unresolved_attempts": [], | |
| "audit_complete": true, | |
| "accounting": "reported-responses" | |
| }, | |
| "codex_wall_time_s": 1973.915146, | |
| "kinex_wall_time_s": 3231.211091, | |
| "kinex_version": "0.10.3", | |
| "kinex_revision": "a1eaece0bd01a25177a43c2091c10b213b416aa5", | |
| "same_roboenv_source": true | |
| }, | |
| { | |
| "task_key": "task01/05", | |
| "instruction": "Add the butter stick, sugar cube, and cream cheese stick to the stand mixer bowl, lower the mixer head fully, and turn the speed knob to begin making cheesecake filling.", | |
| "instruction_policy": "modified", | |
| "codex_success": true, | |
| "kinex_success": false, | |
| "codex_steps": 5224, | |
| "kinex_steps": 5930, | |
| "codex_usage": { | |
| "accounting": "reported-responses", | |
| "audit_complete": true, | |
| "cache_hit_rate": 0.9885663091343345, | |
| "cache_reported_input_tokens": 12355153, | |
| "cache_write_input_tokens": 0, | |
| "cache_write_reported_input_tokens": 12355153, | |
| "cached_input_tokens": 12213888, | |
| "completed_turns": 1, | |
| "cost_usd": null, | |
| "failed_turns": 0, | |
| "input_tokens": 12355153, | |
| "known_cache_write_input_tokens": 0, | |
| "known_cached_input_tokens": 12213888, | |
| "known_input_tokens": 12355153, | |
| "known_output_tokens": 35229, | |
| "known_reasoning_output_tokens": 13787, | |
| "output_tokens": 35229, | |
| "reasoning_output_tokens": 13787, | |
| "reasoning_reported_output_tokens": 35229, | |
| "reported_responses": { | |
| "cache_reported_input_tokens": 202, | |
| "cache_write_input_tokens": 202, | |
| "cache_write_reported_input_tokens": 202, | |
| "cached_input_tokens": 202, | |
| "input_tokens": 202, | |
| "output_tokens": 202, | |
| "reasoning_output_tokens": 202, | |
| "reasoning_reported_output_tokens": 202 | |
| }, | |
| "response_count": 202, | |
| "response_ids_complete": true, | |
| "schema": "rlebench/token-usage/1", | |
| "source": "Codex token_usage_record per response", | |
| "uncached_input_tokens": 141265, | |
| "unidentified_usage_records": 0 | |
| }, | |
| "kinex_usage": { | |
| "input_tokens": 10985080, | |
| "output_tokens": 19054, | |
| "cached_input_tokens": 10880128, | |
| "cache_write_input_tokens": 0, | |
| "reasoning_output_tokens": 6692, | |
| "cache_reported_input_tokens": 10985080, | |
| "cache_write_reported_input_tokens": 10985080, | |
| "reasoning_reported_output_tokens": 19054, | |
| "schema": "rlebench/token-usage/1", | |
| "source": "Provider terminal response usage", | |
| "response_count": 217, | |
| "reported_responses": { | |
| "input_tokens": 217, | |
| "output_tokens": 217, | |
| "cached_input_tokens": 217, | |
| "cache_write_input_tokens": 217, | |
| "reasoning_output_tokens": 217, | |
| "cache_reported_input_tokens": 217, | |
| "cache_write_reported_input_tokens": 217, | |
| "reasoning_reported_output_tokens": 217 | |
| }, | |
| "known_cached_input_tokens": 10880128, | |
| "known_input_tokens": 10985080, | |
| "known_output_tokens": 19054, | |
| "known_cache_write_input_tokens": 0, | |
| "known_reasoning_output_tokens": 6692, | |
| "cache_hit_rate": 0.9904459503253504, | |
| "uncached_input_tokens": 104952, | |
| "cost_usd": null, | |
| "response_ids_complete": true, | |
| "usage_source": "provider-raw", | |
| "request_attempts": 217, | |
| "rejected_attempts": 0, | |
| "unresolved_attempts": [], | |
| "audit_complete": true, | |
| "accounting": "reported-responses" | |
| }, | |
| "codex_wall_time_s": 1964.925029, | |
| "kinex_wall_time_s": 2570.336196, | |
| "kinex_version": "0.10.3", | |
| "kinex_revision": "a1eaece0bd01a25177a43c2091c10b213b416aa5", | |
| "same_roboenv_source": true | |
| }, | |
| { | |
| "task_key": "task01/06", | |
| "instruction": "Turn on the sink faucet. Move the specified vegetable from the counter into the sink while the water is running. Turn off the faucet, move the vegetable into the pot next to the stove, and move the pot to the specified burner.", | |
| "instruction_policy": "modified", | |
| "codex_success": false, | |
| "kinex_success": false, | |
| "codex_steps": 5416, | |
| "kinex_steps": 5856, | |
| "codex_usage": { | |
| "accounting": "reported-responses", | |
| "audit_complete": true, | |
| "cache_hit_rate": 0.9880091550479709, | |
| "cache_reported_input_tokens": 9630097, | |
| "cache_write_input_tokens": 0, | |
| "cache_write_reported_input_tokens": 9630097, | |
| "cached_input_tokens": 9514624, | |
| "completed_turns": 1, | |
| "cost_usd": null, | |
| "failed_turns": 0, | |
| "input_tokens": 9630097, | |
| "known_cache_write_input_tokens": 0, | |
| "known_cached_input_tokens": 9514624, | |
| "known_input_tokens": 9630097, | |
| "known_output_tokens": 24897, | |
| "known_reasoning_output_tokens": 10825, | |
| "output_tokens": 24897, | |
| "reasoning_output_tokens": 10825, | |
| "reasoning_reported_output_tokens": 24897, | |
| "reported_responses": { | |
| "cache_reported_input_tokens": 145, | |
| "cache_write_input_tokens": 145, | |
| "cache_write_reported_input_tokens": 145, | |
| "cached_input_tokens": 145, | |
| "input_tokens": 145, | |
| "output_tokens": 145, | |
| "reasoning_output_tokens": 145, | |
| "reasoning_reported_output_tokens": 145 | |
| }, | |
| "response_count": 145, | |
| "response_ids_complete": true, | |
| "schema": "rlebench/token-usage/1", | |
| "source": "Codex token_usage_record per response", | |
| "uncached_input_tokens": 115473, | |
| "unidentified_usage_records": 0 | |
| }, | |
| "kinex_usage": { | |
| "input_tokens": 26424116, | |
| "output_tokens": 28463, | |
| "cached_input_tokens": 26255616, | |
| "cache_write_input_tokens": 0, | |
| "reasoning_output_tokens": 11086, | |
| "cache_reported_input_tokens": 26424116, | |
| "cache_write_reported_input_tokens": 26424116, | |
| "reasoning_reported_output_tokens": 28463, | |
| "schema": "rlebench/token-usage/1", | |
| "source": "Provider terminal response usage", | |
| "response_count": 347, | |
| "reported_responses": { | |
| "input_tokens": 347, | |
| "output_tokens": 347, | |
| "cached_input_tokens": 347, | |
| "cache_write_input_tokens": 347, | |
| "reasoning_output_tokens": 347, | |
| "cache_reported_input_tokens": 347, | |
| "cache_write_reported_input_tokens": 347, | |
| "reasoning_reported_output_tokens": 347 | |
| }, | |
| "known_cached_input_tokens": 26255616, | |
| "known_input_tokens": 26424116, | |
| "known_output_tokens": 28463, | |
| "known_cache_write_input_tokens": 0, | |
| "known_reasoning_output_tokens": 11086, | |
| "cache_hit_rate": 0.9936232493075643, | |
| "uncached_input_tokens": 168500, | |
| "cost_usd": null, | |
| "response_ids_complete": true, | |
| "usage_source": "provider-raw", | |
| "request_attempts": 347, | |
| "rejected_attempts": 0, | |
| "unresolved_attempts": [], | |
| "audit_complete": true, | |
| "accounting": "reported-responses" | |
| }, | |
| "codex_wall_time_s": 1456.598048, | |
| "kinex_wall_time_s": 3438.331724, | |
| "kinex_version": "0.10.3", | |
| "kinex_revision": "a1eaece0bd01a25177a43c2091c10b213b416aa5", | |
| "same_roboenv_source": true | |
| }, | |
| { | |
| "task_key": "task01/07", | |
| "instruction": "Take the specified meat from the fridge and place it on the digital scale on the counter by the fridge. Release it and move the gripper clear while waiting a few seconds for a reading. Then move it to the plate on the dining counter, release it, and move the gripper clear.", | |
| "instruction_policy": "modified", | |
| "codex_success": false, | |
| "kinex_success": false, | |
| "codex_steps": 1426, | |
| "kinex_steps": 2427, | |
| "codex_usage": { | |
| "accounting": "reported-responses", | |
| "audit_complete": true, | |
| "cache_hit_rate": 0.9663948735854094, | |
| "cache_reported_input_tokens": 1217225, | |
| "cache_write_input_tokens": 0, | |
| "cache_write_reported_input_tokens": 1217225, | |
| "cached_input_tokens": 1176320, | |
| "completed_turns": 1, | |
| "cost_usd": null, | |
| "failed_turns": 0, | |
| "input_tokens": 1217225, | |
| "known_cache_write_input_tokens": 0, | |
| "known_cached_input_tokens": 1176320, | |
| "known_input_tokens": 1217225, | |
| "known_output_tokens": 8600, | |
| "known_reasoning_output_tokens": 2758, | |
| "output_tokens": 8600, | |
| "reasoning_output_tokens": 2758, | |
| "reasoning_reported_output_tokens": 8600, | |
| "reported_responses": { | |
| "cache_reported_input_tokens": 36, | |
| "cache_write_input_tokens": 36, | |
| "cache_write_reported_input_tokens": 36, | |
| "cached_input_tokens": 36, | |
| "input_tokens": 36, | |
| "output_tokens": 36, | |
| "reasoning_output_tokens": 36, | |
| "reasoning_reported_output_tokens": 36 | |
| }, | |
| "response_count": 36, | |
| "response_ids_complete": true, | |
| "schema": "rlebench/token-usage/1", | |
| "source": "Codex token_usage_record per response", | |
| "uncached_input_tokens": 40905, | |
| "unidentified_usage_records": 0 | |
| }, | |
| "kinex_usage": { | |
| "input_tokens": 7005688, | |
| "output_tokens": 18192, | |
| "cached_input_tokens": 6921216, | |
| "cache_write_input_tokens": 0, | |
| "reasoning_output_tokens": 6177, | |
| "cache_reported_input_tokens": 7005688, | |
| "cache_write_reported_input_tokens": 7005688, | |
| "reasoning_reported_output_tokens": 18192, | |
| "schema": "rlebench/token-usage/1", | |
| "source": "Provider terminal response usage", | |
| "response_count": 153, | |
| "reported_responses": { | |
| "input_tokens": 153, | |
| "output_tokens": 153, | |
| "cached_input_tokens": 153, | |
| "cache_write_input_tokens": 153, | |
| "reasoning_output_tokens": 153, | |
| "cache_reported_input_tokens": 153, | |
| "cache_write_reported_input_tokens": 153, | |
| "reasoning_reported_output_tokens": 153 | |
| }, | |
| "known_cached_input_tokens": 6921216, | |
| "known_input_tokens": 7005688, | |
| "known_output_tokens": 18192, | |
| "known_cache_write_input_tokens": 0, | |
| "known_reasoning_output_tokens": 6177, | |
| "cache_hit_rate": 0.987942369114925, | |
| "uncached_input_tokens": 84472, | |
| "cost_usd": null, | |
| "response_ids_complete": true, | |
| "usage_source": "provider-raw", | |
| "request_attempts": 155, | |
| "rejected_attempts": 0, | |
| "unresolved_attempts": [], | |
| "audit_complete": true, | |
| "accounting": "reported-responses" | |
| }, | |
| "codex_wall_time_s": 394.373333, | |
| "kinex_wall_time_s": 1396.812595, | |
| "kinex_version": "0.10.3", | |
| "kinex_revision": "a1eaece0bd01a25177a43c2091c10b213b416aa5", | |
| "same_roboenv_source": true | |
| }, | |
| { | |
| "task_key": "task01/08", | |
| "instruction": "Pick up the sponge from the counter and scrub across a broad area of the cutting board, keeping the sponge grasped and in contact with the board throughout the scrubbing motion. Once finished, release the sponge and retract the gripper well away from it.", | |
| "instruction_policy": "modified", | |
| "codex_success": false, | |
| "kinex_success": true, | |
| "codex_steps": 1212, | |
| "kinex_steps": 1007, | |
| "codex_usage": { | |
| "accounting": "reported-responses", | |
| "audit_complete": true, | |
| "cache_hit_rate": 0.9606465096549688, | |
| "cache_reported_input_tokens": 775611, | |
| "cache_write_input_tokens": 0, | |
| "cache_write_reported_input_tokens": 775611, | |
| "cached_input_tokens": 745088, | |
| "completed_turns": 1, | |
| "cost_usd": null, | |
| "failed_turns": 0, | |
| "input_tokens": 775611, | |
| "known_cache_write_input_tokens": 0, | |
| "known_cached_input_tokens": 745088, | |
| "known_input_tokens": 775611, | |
| "known_output_tokens": 6607, | |
| "known_reasoning_output_tokens": 1428, | |
| "output_tokens": 6607, | |
| "reasoning_output_tokens": 1428, | |
| "reasoning_reported_output_tokens": 6607, | |
| "reported_responses": { | |
| "cache_reported_input_tokens": 26, | |
| "cache_write_input_tokens": 26, | |
| "cache_write_reported_input_tokens": 26, | |
| "cached_input_tokens": 26, | |
| "input_tokens": 26, | |
| "output_tokens": 26, | |
| "reasoning_output_tokens": 26, | |
| "reasoning_reported_output_tokens": 26 | |
| }, | |
| "response_count": 26, | |
| "response_ids_complete": true, | |
| "schema": "rlebench/token-usage/1", | |
| "source": "Codex token_usage_record per response", | |
| "uncached_input_tokens": 30523, | |
| "unidentified_usage_records": 0 | |
| }, | |
| "kinex_usage": { | |
| "input_tokens": 1225659, | |
| "output_tokens": 6356, | |
| "cached_input_tokens": 1182720, | |
| "cache_write_input_tokens": 0, | |
| "reasoning_output_tokens": 1012, | |
| "cache_reported_input_tokens": 1225659, | |
| "cache_write_reported_input_tokens": 1225659, | |
| "reasoning_reported_output_tokens": 6356, | |
| "schema": "rlebench/token-usage/1", | |
| "source": "Provider terminal response usage", | |
| "response_count": 42, | |
| "reported_responses": { | |
| "input_tokens": 42, | |
| "output_tokens": 42, | |
| "cached_input_tokens": 42, | |
| "cache_write_input_tokens": 42, | |
| "reasoning_output_tokens": 42, | |
| "cache_reported_input_tokens": 42, | |
| "cache_write_reported_input_tokens": 42, | |
| "reasoning_reported_output_tokens": 42 | |
| }, | |
| "known_cached_input_tokens": 1182720, | |
| "known_input_tokens": 1225659, | |
| "known_output_tokens": 6356, | |
| "known_cache_write_input_tokens": 0, | |
| "known_reasoning_output_tokens": 1012, | |
| "cache_hit_rate": 0.9649666016404237, | |
| "uncached_input_tokens": 42939, | |
| "cost_usd": null, | |
| "response_ids_complete": true, | |
| "usage_source": "provider-raw", | |
| "request_attempts": 42, | |
| "rejected_attempts": 0, | |
| "unresolved_attempts": [], | |
| "audit_complete": true, | |
| "accounting": "reported-responses" | |
| }, | |
| "codex_wall_time_s": 276.421696, | |
| "kinex_wall_time_s": 418.12241, | |
| "kinex_version": "0.10.3", | |
| "kinex_revision": "a1eaece0bd01a25177a43c2091c10b213b416aa5", | |
| "same_roboenv_source": true | |
| }, | |
| { | |
| "task_key": "task01/09", | |
| "instruction": "Pick the specified vegetable and the cream cheese from the fridge, place both fully inside the blender, and turn it on.", | |
| "instruction_policy": "modified", | |
| "codex_success": false, | |
| "kinex_success": false, | |
| "codex_steps": 2216, | |
| "kinex_steps": 4628, | |
| "codex_usage": { | |
| "accounting": "reported-responses", | |
| "audit_complete": true, | |
| "cache_hit_rate": 0.9733556695336862, | |
| "cache_reported_input_tokens": 1973478, | |
| "cache_write_input_tokens": 0, | |
| "cache_write_reported_input_tokens": 1973478, | |
| "cached_input_tokens": 1920896, | |
| "completed_turns": 1, | |
| "cost_usd": null, | |
| "failed_turns": 0, | |
| "input_tokens": 1973478, | |
| "known_cache_write_input_tokens": 0, | |
| "known_cached_input_tokens": 1920896, | |
| "known_input_tokens": 1973478, | |
| "known_output_tokens": 11695, | |
| "known_reasoning_output_tokens": 4578, | |
| "output_tokens": 11695, | |
| "reasoning_output_tokens": 4578, | |
| "reasoning_reported_output_tokens": 11695, | |
| "reported_responses": { | |
| "cache_reported_input_tokens": 48, | |
| "cache_write_input_tokens": 48, | |
| "cache_write_reported_input_tokens": 48, | |
| "cached_input_tokens": 48, | |
| "input_tokens": 48, | |
| "output_tokens": 48, | |
| "reasoning_output_tokens": 48, | |
| "reasoning_reported_output_tokens": 48 | |
| }, | |
| "response_count": 48, | |
| "response_ids_complete": true, | |
| "schema": "rlebench/token-usage/1", | |
| "source": "Codex token_usage_record per response", | |
| "uncached_input_tokens": 52582, | |
| "unidentified_usage_records": 0 | |
| }, | |
| "kinex_usage": { | |
| "input_tokens": 19377448, | |
| "output_tokens": 36860, | |
| "cached_input_tokens": 19235968, | |
| "cache_write_input_tokens": 0, | |
| "reasoning_output_tokens": 16258, | |
| "cache_reported_input_tokens": 19377448, | |
| "cache_write_reported_input_tokens": 19377448, | |
| "reasoning_reported_output_tokens": 36860, | |
| "schema": "rlebench/token-usage/1", | |
| "source": "Provider terminal response usage", | |
| "response_count": 308, | |
| "reported_responses": { | |
| "input_tokens": 308, | |
| "output_tokens": 308, | |
| "cached_input_tokens": 308, | |
| "cache_write_input_tokens": 308, | |
| "reasoning_output_tokens": 308, | |
| "cache_reported_input_tokens": 308, | |
| "cache_write_reported_input_tokens": 308, | |
| "reasoning_reported_output_tokens": 308 | |
| }, | |
| "known_cached_input_tokens": 19235968, | |
| "known_input_tokens": 19377448, | |
| "known_output_tokens": 36860, | |
| "known_cache_write_input_tokens": 0, | |
| "known_reasoning_output_tokens": 16258, | |
| "cache_hit_rate": 0.9926987289554331, | |
| "uncached_input_tokens": 141480, | |
| "cost_usd": null, | |
| "response_ids_complete": true, | |
| "usage_source": "provider-raw", | |
| "request_attempts": 308, | |
| "rejected_attempts": 0, | |
| "unresolved_attempts": [], | |
| "audit_complete": true, | |
| "accounting": "reported-responses" | |
| }, | |
| "codex_wall_time_s": 625.032726, | |
| "kinex_wall_time_s": 3327.021406, | |
| "kinex_version": "0.10.3", | |
| "kinex_revision": "a1eaece0bd01a25177a43c2091c10b213b416aa5", | |
| "same_roboenv_source": true | |
| }, | |
| { | |
| "task_key": "task01/10", | |
| "instruction": "Pick the specified vegetable from the fridge and hold it under running water from the sink faucet to wash it. Then place it on the tray next to the sink to prepare for roasting. Release it and move the gripper clear.", | |
| "instruction_policy": "modified", | |
| "codex_success": true, | |
| "kinex_success": false, | |
| "codex_steps": 3969, | |
| "kinex_steps": 4505, | |
| "codex_usage": { | |
| "accounting": "reported-responses", | |
| "audit_complete": true, | |
| "cache_hit_rate": 0.9777504182000669, | |
| "cache_reported_input_tokens": 2989000, | |
| "cache_write_input_tokens": 0, | |
| "cache_write_reported_input_tokens": 2989000, | |
| "cached_input_tokens": 2922496, | |
| "completed_turns": 1, | |
| "cost_usd": null, | |
| "failed_turns": 0, | |
| "input_tokens": 2989000, | |
| "known_cache_write_input_tokens": 0, | |
| "known_cached_input_tokens": 2922496, | |
| "known_input_tokens": 2989000, | |
| "known_output_tokens": 13452, | |
| "known_reasoning_output_tokens": 4273, | |
| "output_tokens": 13452, | |
| "reasoning_output_tokens": 4273, | |
| "reasoning_reported_output_tokens": 13452, | |
| "reported_responses": { | |
| "cache_reported_input_tokens": 68, | |
| "cache_write_input_tokens": 68, | |
| "cache_write_reported_input_tokens": 68, | |
| "cached_input_tokens": 68, | |
| "input_tokens": 68, | |
| "output_tokens": 68, | |
| "reasoning_output_tokens": 68, | |
| "reasoning_reported_output_tokens": 68 | |
| }, | |
| "response_count": 68, | |
| "response_ids_complete": true, | |
| "schema": "rlebench/token-usage/1", | |
| "source": "Codex token_usage_record per response", | |
| "uncached_input_tokens": 66504, | |
| "unidentified_usage_records": 0 | |
| }, | |
| "kinex_usage": { | |
| "input_tokens": 8091847, | |
| "output_tokens": 21393, | |
| "cached_input_tokens": 7916288, | |
| "cache_write_input_tokens": 0, | |
| "reasoning_output_tokens": 8065, | |
| "cache_reported_input_tokens": 8091847, | |
| "cache_write_reported_input_tokens": 8091847, | |
| "reasoning_reported_output_tokens": 21393, | |
| "schema": "rlebench/token-usage/1", | |
| "source": "Provider terminal response usage", | |
| "response_count": 160, | |
| "reported_responses": { | |
| "input_tokens": 160, | |
| "output_tokens": 160, | |
| "cached_input_tokens": 160, | |
| "cache_write_input_tokens": 160, | |
| "reasoning_output_tokens": 160, | |
| "cache_reported_input_tokens": 160, | |
| "cache_write_reported_input_tokens": 160, | |
| "reasoning_reported_output_tokens": 160 | |
| }, | |
| "known_cached_input_tokens": 7916288, | |
| "known_input_tokens": 8091847, | |
| "known_output_tokens": 21393, | |
| "known_cache_write_input_tokens": 0, | |
| "known_reasoning_output_tokens": 8065, | |
| "cache_hit_rate": 0.9783042116342536, | |
| "uncached_input_tokens": 175559, | |
| "cost_usd": null, | |
| "response_ids_complete": true, | |
| "usage_source": "provider-raw", | |
| "request_attempts": 160, | |
| "rejected_attempts": 0, | |
| "unresolved_attempts": [], | |
| "audit_complete": true, | |
| "accounting": "reported-responses" | |
| }, | |
| "codex_wall_time_s": 800.707309, | |
| "kinex_wall_time_s": 1708.264043, | |
| "kinex_version": "0.10.3", | |
| "kinex_revision": "a1eaece0bd01a25177a43c2091c10b213b416aa5", | |
| "same_roboenv_source": true | |
| }, | |
| { | |
| "task_key": "task02/03", | |
| "instruction": "turn on the stove and put the moka pot on it", | |
| "instruction_policy": "original_native", | |
| "codex_success": true, | |
| "kinex_success": true, | |
| "codex_steps": 2013, | |
| "kinex_steps": 706, | |
| "codex_usage": { | |
| "accounting": "reported-responses", | |
| "audit_complete": true, | |
| "cache_hit_rate": 0.967178914688548, | |
| "cache_reported_input_tokens": 2225094, | |
| "cache_write_input_tokens": 0, | |
| "cache_write_reported_input_tokens": 2225094, | |
| "cached_input_tokens": 2152064, | |
| "completed_turns": 1, | |
| "cost_usd": null, | |
| "failed_turns": 0, | |
| "input_tokens": 2225094, | |
| "known_cache_write_input_tokens": 0, | |
| "known_cached_input_tokens": 2152064, | |
| "known_input_tokens": 2225094, | |
| "known_output_tokens": 13673, | |
| "known_reasoning_output_tokens": 5025, | |
| "output_tokens": 13673, | |
| "reasoning_output_tokens": 5025, | |
| "reasoning_reported_output_tokens": 13673, | |
| "reported_responses": { | |
| "cache_reported_input_tokens": 50, | |
| "cache_write_input_tokens": 50, | |
| "cache_write_reported_input_tokens": 50, | |
| "cached_input_tokens": 50, | |
| "input_tokens": 50, | |
| "output_tokens": 50, | |
| "reasoning_output_tokens": 50, | |
| "reasoning_reported_output_tokens": 50 | |
| }, | |
| "response_count": 50, | |
| "response_ids_complete": true, | |
| "schema": "rlebench/token-usage/1", | |
| "source": "Codex token_usage_record per response", | |
| "uncached_input_tokens": 73030, | |
| "unidentified_usage_records": 0 | |
| }, | |
| "kinex_usage": { | |
| "input_tokens": 1221677, | |
| "output_tokens": 6604, | |
| "cached_input_tokens": 1146496, | |
| "cache_write_input_tokens": 0, | |
| "reasoning_output_tokens": 1955, | |
| "cache_reported_input_tokens": 1221677, | |
| "cache_write_reported_input_tokens": 1221677, | |
| "reasoning_reported_output_tokens": 6604, | |
| "schema": "rlebench/token-usage/1", | |
| "source": "Provider terminal response usage", | |
| "response_count": 47, | |
| "reported_responses": { | |
| "input_tokens": 47, | |
| "output_tokens": 47, | |
| "cached_input_tokens": 47, | |
| "cache_write_input_tokens": 47, | |
| "reasoning_output_tokens": 47, | |
| "cache_reported_input_tokens": 47, | |
| "cache_write_reported_input_tokens": 47, | |
| "reasoning_reported_output_tokens": 47 | |
| }, | |
| "known_cached_input_tokens": 1146496, | |
| "known_input_tokens": 1221677, | |
| "known_output_tokens": 6604, | |
| "known_cache_write_input_tokens": 0, | |
| "known_reasoning_output_tokens": 1955, | |
| "cache_hit_rate": 0.9384608206588158, | |
| "uncached_input_tokens": 75181, | |
| "cost_usd": null, | |
| "response_ids_complete": true, | |
| "accounting": "reported-responses", | |
| "usage_source": "provider-raw", | |
| "request_attempts": 47, | |
| "rejected_attempts": 0, | |
| "unresolved_attempts": [], | |
| "audit_complete": true | |
| }, | |
| "codex_wall_time_s": 531.974225, | |
| "kinex_wall_time_s": 371.824558, | |
| "kinex_version": "0.10.3", | |
| "kinex_revision": "caac19a8a36272972f762e0f74cfe381e8500048", | |
| "same_roboenv_source": true | |
| }, | |
| { | |
| "task_key": "task02/01", | |
| "instruction": "put both the alphabet soup and the tomato sauce in the basket", | |
| "instruction_policy": "original_native", | |
| "codex_success": true, | |
| "kinex_success": true, | |
| "codex_steps": 969, | |
| "kinex_steps": 1094, | |
| "codex_usage": { | |
| "accounting": "reported-responses", | |
| "audit_complete": true, | |
| "cache_hit_rate": 0.9509386255123954, | |
| "cache_reported_input_tokens": 832121, | |
| "cache_write_input_tokens": 0, | |
| "cache_write_reported_input_tokens": 832121, | |
| "cached_input_tokens": 791296, | |
| "completed_turns": 1, | |
| "cost_usd": null, | |
| "failed_turns": 0, | |
| "input_tokens": 832121, | |
| "known_cache_write_input_tokens": 0, | |
| "known_cached_input_tokens": 791296, | |
| "known_input_tokens": 832121, | |
| "known_output_tokens": 6802, | |
| "known_reasoning_output_tokens": 1635, | |
| "output_tokens": 6802, | |
| "reasoning_output_tokens": 1635, | |
| "reasoning_reported_output_tokens": 6802, | |
| "reported_responses": { | |
| "cache_reported_input_tokens": 27, | |
| "cache_write_input_tokens": 27, | |
| "cache_write_reported_input_tokens": 27, | |
| "cached_input_tokens": 27, | |
| "input_tokens": 27, | |
| "output_tokens": 27, | |
| "reasoning_output_tokens": 27, | |
| "reasoning_reported_output_tokens": 27 | |
| }, | |
| "response_count": 27, | |
| "response_ids_complete": true, | |
| "schema": "rlebench/token-usage/1", | |
| "source": "Codex token_usage_record per response", | |
| "uncached_input_tokens": 40825, | |
| "unidentified_usage_records": 0 | |
| }, | |
| "kinex_usage": { | |
| "input_tokens": 1546168, | |
| "output_tokens": 7604, | |
| "cached_input_tokens": 1461888, | |
| "cache_write_input_tokens": 0, | |
| "reasoning_output_tokens": 2069, | |
| "cache_reported_input_tokens": 1546168, | |
| "cache_write_reported_input_tokens": 1546168, | |
| "reasoning_reported_output_tokens": 7604, | |
| "schema": "rlebench/token-usage/1", | |
| "source": "Provider terminal response usage", | |
| "response_count": 55, | |
| "reported_responses": { | |
| "input_tokens": 55, | |
| "output_tokens": 55, | |
| "cached_input_tokens": 55, | |
| "cache_write_input_tokens": 55, | |
| "reasoning_output_tokens": 55, | |
| "cache_reported_input_tokens": 55, | |
| "cache_write_reported_input_tokens": 55, | |
| "reasoning_reported_output_tokens": 55 | |
| }, | |
| "known_cached_input_tokens": 1461888, | |
| "known_input_tokens": 1546168, | |
| "known_output_tokens": 7604, | |
| "known_cache_write_input_tokens": 0, | |
| "known_reasoning_output_tokens": 2069, | |
| "cache_hit_rate": 0.9454910462511189, | |
| "uncached_input_tokens": 84280, | |
| "cost_usd": null, | |
| "response_ids_complete": true, | |
| "accounting": "reported-responses", | |
| "usage_source": "provider-raw", | |
| "request_attempts": 55, | |
| "rejected_attempts": 0, | |
| "unresolved_attempts": [], | |
| "audit_complete": true | |
| }, | |
| "codex_wall_time_s": 272.779105, | |
| "kinex_wall_time_s": 406.249632, | |
| "kinex_version": "0.10.3", | |
| "kinex_revision": "caac19a8a36272972f762e0f74cfe381e8500048", | |
| "same_roboenv_source": true | |
| }, | |
| { | |
| "task_key": "task02/02", | |
| "instruction": "put both the cream cheese box and the butter in the basket", | |
| "instruction_policy": "original_native", | |
| "codex_success": true, | |
| "kinex_success": true, | |
| "codex_steps": 722, | |
| "kinex_steps": 1011, | |
| "codex_usage": { | |
| "accounting": "reported-responses", | |
| "audit_complete": true, | |
| "cache_hit_rate": 0.9515345728205089, | |
| "cache_reported_input_tokens": 784518, | |
| "cache_write_input_tokens": 0, | |
| "cache_write_reported_input_tokens": 784518, | |
| "cached_input_tokens": 746496, | |
| "completed_turns": 1, | |
| "cost_usd": null, | |
| "failed_turns": 0, | |
| "input_tokens": 784518, | |
| "known_cache_write_input_tokens": 0, | |
| "known_cached_input_tokens": 746496, | |
| "known_input_tokens": 784518, | |
| "known_output_tokens": 6225, | |
| "known_reasoning_output_tokens": 1371, | |
| "output_tokens": 6225, | |
| "reasoning_output_tokens": 1371, | |
| "reasoning_reported_output_tokens": 6225, | |
| "reported_responses": { | |
| "cache_reported_input_tokens": 28, | |
| "cache_write_input_tokens": 28, | |
| "cache_write_reported_input_tokens": 28, | |
| "cached_input_tokens": 28, | |
| "input_tokens": 28, | |
| "output_tokens": 28, | |
| "reasoning_output_tokens": 28, | |
| "reasoning_reported_output_tokens": 28 | |
| }, | |
| "response_count": 28, | |
| "response_ids_complete": true, | |
| "schema": "rlebench/token-usage/1", | |
| "source": "Codex token_usage_record per response", | |
| "uncached_input_tokens": 38022, | |
| "unidentified_usage_records": 0 | |
| }, | |
| "kinex_usage": { | |
| "input_tokens": 2453051, | |
| "output_tokens": 9069, | |
| "cached_input_tokens": 2363776, | |
| "cache_write_input_tokens": 0, | |
| "reasoning_output_tokens": 2858, | |
| "cache_reported_input_tokens": 2453051, | |
| "cache_write_reported_input_tokens": 2453051, | |
| "reasoning_reported_output_tokens": 9069, | |
| "schema": "rlebench/token-usage/1", | |
| "source": "Provider terminal response usage", | |
| "response_count": 84, | |
| "reported_responses": { | |
| "input_tokens": 84, | |
| "output_tokens": 84, | |
| "cached_input_tokens": 84, | |
| "cache_write_input_tokens": 84, | |
| "reasoning_output_tokens": 84, | |
| "cache_reported_input_tokens": 84, | |
| "cache_write_reported_input_tokens": 84, | |
| "reasoning_reported_output_tokens": 84 | |
| }, | |
| "known_cached_input_tokens": 2363776, | |
| "known_input_tokens": 2453051, | |
| "known_output_tokens": 9069, | |
| "known_cache_write_input_tokens": 0, | |
| "known_reasoning_output_tokens": 2858, | |
| "cache_hit_rate": 0.9636065454815248, | |
| "uncached_input_tokens": 89275, | |
| "cost_usd": null, | |
| "response_ids_complete": true, | |
| "accounting": "reported-responses", | |
| "usage_source": "provider-raw", | |
| "request_attempts": 84, | |
| "rejected_attempts": 0, | |
| "unresolved_attempts": [], | |
| "audit_complete": true | |
| }, | |
| "codex_wall_time_s": 231.024585, | |
| "kinex_wall_time_s": 611.087641, | |
| "kinex_version": "0.10.3", | |
| "kinex_revision": "caac19a8a36272972f762e0f74cfe381e8500048", | |
| "same_roboenv_source": true | |
| }, | |
| { | |
| "task_key": "task02/04", | |
| "instruction": "put the black bowl in the bottom drawer of the cabinet and close it", | |
| "instruction_policy": "original_native", | |
| "codex_success": false, | |
| "kinex_success": true, | |
| "codex_steps": 3687, | |
| "kinex_steps": 1483, | |
| "codex_usage": { | |
| "accounting": "reported-responses", | |
| "audit_complete": true, | |
| "cache_hit_rate": 0.9745746678635921, | |
| "cache_reported_input_tokens": 3356377, | |
| "cache_write_input_tokens": 0, | |
| "cache_write_reported_input_tokens": 3356377, | |
| "cached_input_tokens": 3271040, | |
| "completed_turns": 1, | |
| "cost_usd": null, | |
| "failed_turns": 0, | |
| "input_tokens": 3356377, | |
| "known_cache_write_input_tokens": 0, | |
| "known_cached_input_tokens": 3271040, | |
| "known_input_tokens": 3356377, | |
| "known_output_tokens": 23446, | |
| "known_reasoning_output_tokens": 11600, | |
| "output_tokens": 23446, | |
| "reasoning_output_tokens": 11600, | |
| "reasoning_reported_output_tokens": 23446, | |
| "reported_responses": { | |
| "cache_reported_input_tokens": 67, | |
| "cache_write_input_tokens": 67, | |
| "cache_write_reported_input_tokens": 67, | |
| "cached_input_tokens": 67, | |
| "input_tokens": 67, | |
| "output_tokens": 67, | |
| "reasoning_output_tokens": 67, | |
| "reasoning_reported_output_tokens": 67 | |
| }, | |
| "response_count": 67, | |
| "response_ids_complete": true, | |
| "schema": "rlebench/token-usage/1", | |
| "source": "Codex token_usage_record per response", | |
| "uncached_input_tokens": 85337, | |
| "unidentified_usage_records": 0 | |
| }, | |
| "kinex_usage": { | |
| "input_tokens": 1990960, | |
| "output_tokens": 12374, | |
| "cached_input_tokens": 1883904, | |
| "cache_write_input_tokens": 0, | |
| "reasoning_output_tokens": 5081, | |
| "cache_reported_input_tokens": 1990960, | |
| "cache_write_reported_input_tokens": 1990960, | |
| "reasoning_reported_output_tokens": 12374, | |
| "schema": "rlebench/token-usage/1", | |
| "source": "Provider terminal response usage", | |
| "response_count": 59, | |
| "reported_responses": { | |
| "input_tokens": 59, | |
| "output_tokens": 59, | |
| "cached_input_tokens": 59, | |
| "cache_write_input_tokens": 59, | |
| "reasoning_output_tokens": 59, | |
| "cache_reported_input_tokens": 59, | |
| "cache_write_reported_input_tokens": 59, | |
| "reasoning_reported_output_tokens": 59 | |
| }, | |
| "known_cached_input_tokens": 1883904, | |
| "known_input_tokens": 1990960, | |
| "known_output_tokens": 12374, | |
| "known_cache_write_input_tokens": 0, | |
| "known_reasoning_output_tokens": 5081, | |
| "cache_hit_rate": 0.9462289548760398, | |
| "uncached_input_tokens": 107056, | |
| "cost_usd": null, | |
| "response_ids_complete": true, | |
| "accounting": "reported-responses", | |
| "usage_source": "provider-raw", | |
| "request_attempts": 59, | |
| "rejected_attempts": 0, | |
| "unresolved_attempts": [], | |
| "audit_complete": true | |
| }, | |
| "codex_wall_time_s": 919.204412, | |
| "kinex_wall_time_s": 624.888736, | |
| "kinex_version": "0.10.3", | |
| "kinex_revision": "caac19a8a36272972f762e0f74cfe381e8500048", | |
| "same_roboenv_source": true | |
| }, | |
| { | |
| "task_key": "task02/05", | |
| "instruction": "put the white mug in the center of the left plate and put the yellow and white mug in the center of the right plate, with each mug resting on its plate", | |
| "instruction_policy": "modified", | |
| "codex_success": true, | |
| "kinex_success": true, | |
| "codex_steps": 634, | |
| "kinex_steps": 765, | |
| "codex_usage": { | |
| "accounting": "reported-responses", | |
| "audit_complete": true, | |
| "cache_hit_rate": 0.9493350495033964, | |
| "cache_reported_input_tokens": 509662, | |
| "cache_write_input_tokens": 0, | |
| "cache_write_reported_input_tokens": 509662, | |
| "cached_input_tokens": 483840, | |
| "completed_turns": 1, | |
| "cost_usd": null, | |
| "failed_turns": 0, | |
| "input_tokens": 509662, | |
| "known_cache_write_input_tokens": 0, | |
| "known_cached_input_tokens": 483840, | |
| "known_input_tokens": 509662, | |
| "known_output_tokens": 5212, | |
| "known_reasoning_output_tokens": 1183, | |
| "output_tokens": 5212, | |
| "reasoning_output_tokens": 1183, | |
| "reasoning_reported_output_tokens": 5212, | |
| "reported_responses": { | |
| "cache_reported_input_tokens": 20, | |
| "cache_write_input_tokens": 20, | |
| "cache_write_reported_input_tokens": 20, | |
| "cached_input_tokens": 20, | |
| "input_tokens": 20, | |
| "output_tokens": 20, | |
| "reasoning_output_tokens": 20, | |
| "reasoning_reported_output_tokens": 20 | |
| }, | |
| "response_count": 20, | |
| "response_ids_complete": true, | |
| "schema": "rlebench/token-usage/1", | |
| "source": "Codex token_usage_record per response", | |
| "uncached_input_tokens": 25822, | |
| "unidentified_usage_records": 0 | |
| }, | |
| "kinex_usage": { | |
| "input_tokens": 2532606, | |
| "output_tokens": 7281, | |
| "cached_input_tokens": 2444032, | |
| "cache_write_input_tokens": 0, | |
| "reasoning_output_tokens": 1949, | |
| "cache_reported_input_tokens": 2532606, | |
| "cache_write_reported_input_tokens": 2532606, | |
| "reasoning_reported_output_tokens": 7281, | |
| "schema": "rlebench/token-usage/1", | |
| "source": "Provider terminal response usage", | |
| "response_count": 88, | |
| "reported_responses": { | |
| "input_tokens": 88, | |
| "output_tokens": 88, | |
| "cached_input_tokens": 88, | |
| "cache_write_input_tokens": 88, | |
| "reasoning_output_tokens": 88, | |
| "cache_reported_input_tokens": 88, | |
| "cache_write_reported_input_tokens": 88, | |
| "reasoning_reported_output_tokens": 88 | |
| }, | |
| "known_cached_input_tokens": 2444032, | |
| "known_input_tokens": 2532606, | |
| "known_output_tokens": 7281, | |
| "known_cache_write_input_tokens": 0, | |
| "known_reasoning_output_tokens": 1949, | |
| "cache_hit_rate": 0.9650265378823236, | |
| "uncached_input_tokens": 88574, | |
| "cost_usd": null, | |
| "response_ids_complete": true, | |
| "accounting": "reported-responses", | |
| "usage_source": "provider-raw", | |
| "request_attempts": 88, | |
| "rejected_attempts": 0, | |
| "unresolved_attempts": [], | |
| "audit_complete": true | |
| }, | |
| "codex_wall_time_s": 195.147246, | |
| "kinex_wall_time_s": 569.224064, | |
| "kinex_version": "0.10.3", | |
| "kinex_revision": "caac19a8a36272972f762e0f74cfe381e8500048", | |
| "same_roboenv_source": true | |
| }, | |
| { | |
| "task_key": "task02/06", | |
| "instruction": "pick up the book and place it in the back compartment of the caddy, between the two large side compartments", | |
| "instruction_policy": "modified", | |
| "codex_success": true, | |
| "kinex_success": true, | |
| "codex_steps": 325, | |
| "kinex_steps": 441, | |
| "codex_usage": { | |
| "accounting": "reported-responses", | |
| "audit_complete": true, | |
| "cache_hit_rate": 0.956185574299006, | |
| "cache_reported_input_tokens": 707210, | |
| "cache_write_input_tokens": 0, | |
| "cache_write_reported_input_tokens": 707210, | |
| "cached_input_tokens": 676224, | |
| "completed_turns": 1, | |
| "cost_usd": null, | |
| "failed_turns": 0, | |
| "input_tokens": 707210, | |
| "known_cache_write_input_tokens": 0, | |
| "known_cached_input_tokens": 676224, | |
| "known_input_tokens": 707210, | |
| "known_output_tokens": 6550, | |
| "known_reasoning_output_tokens": 2424, | |
| "output_tokens": 6550, | |
| "reasoning_output_tokens": 2424, | |
| "reasoning_reported_output_tokens": 6550, | |
| "reported_responses": { | |
| "cache_reported_input_tokens": 25, | |
| "cache_write_input_tokens": 25, | |
| "cache_write_reported_input_tokens": 25, | |
| "cached_input_tokens": 25, | |
| "input_tokens": 25, | |
| "output_tokens": 25, | |
| "reasoning_output_tokens": 25, | |
| "reasoning_reported_output_tokens": 25 | |
| }, | |
| "response_count": 25, | |
| "response_ids_complete": true, | |
| "schema": "rlebench/token-usage/1", | |
| "source": "Codex token_usage_record per response", | |
| "uncached_input_tokens": 30986, | |
| "unidentified_usage_records": 0 | |
| }, | |
| "kinex_usage": { | |
| "input_tokens": 1345973, | |
| "output_tokens": 6571, | |
| "cached_input_tokens": 1272960, | |
| "cache_write_input_tokens": 0, | |
| "reasoning_output_tokens": 2281, | |
| "cache_reported_input_tokens": 1345973, | |
| "cache_write_reported_input_tokens": 1345973, | |
| "reasoning_reported_output_tokens": 6571, | |
| "schema": "rlebench/token-usage/1", | |
| "source": "Provider terminal response usage", | |
| "response_count": 54, | |
| "reported_responses": { | |
| "input_tokens": 54, | |
| "output_tokens": 54, | |
| "cached_input_tokens": 54, | |
| "cache_write_input_tokens": 54, | |
| "reasoning_output_tokens": 54, | |
| "cache_reported_input_tokens": 54, | |
| "cache_write_reported_input_tokens": 54, | |
| "reasoning_reported_output_tokens": 54 | |
| }, | |
| "known_cached_input_tokens": 1272960, | |
| "known_input_tokens": 1345973, | |
| "known_output_tokens": 6571, | |
| "known_cache_write_input_tokens": 0, | |
| "known_reasoning_output_tokens": 2281, | |
| "cache_hit_rate": 0.9457544839309555, | |
| "uncached_input_tokens": 73013, | |
| "cost_usd": null, | |
| "response_ids_complete": true, | |
| "accounting": "reported-responses", | |
| "usage_source": "provider-raw", | |
| "request_attempts": 54, | |
| "rejected_attempts": 0, | |
| "unresolved_attempts": [], | |
| "audit_complete": true | |
| }, | |
| "codex_wall_time_s": 241.745547, | |
| "kinex_wall_time_s": 358.977488, | |
| "kinex_version": "0.10.3", | |
| "kinex_revision": "caac19a8a36272972f762e0f74cfe381e8500048", | |
| "same_roboenv_source": true | |
| }, | |
| { | |
| "task_key": "task02/07", | |
| "instruction": "put the white mug in the center of the plate and put the chocolate pudding immediately to the right of the plate", | |
| "instruction_policy": "modified", | |
| "codex_success": true, | |
| "kinex_success": true, | |
| "codex_steps": 1016, | |
| "kinex_steps": 1153, | |
| "codex_usage": { | |
| "accounting": "reported-responses", | |
| "audit_complete": true, | |
| "cache_hit_rate": 0.9487277032832223, | |
| "cache_reported_input_tokens": 701841, | |
| "cache_write_input_tokens": 0, | |
| "cache_write_reported_input_tokens": 701841, | |
| "cached_input_tokens": 665856, | |
| "completed_turns": 1, | |
| "cost_usd": null, | |
| "failed_turns": 0, | |
| "input_tokens": 701841, | |
| "known_cache_write_input_tokens": 0, | |
| "known_cached_input_tokens": 665856, | |
| "known_input_tokens": 701841, | |
| "known_output_tokens": 6754, | |
| "known_reasoning_output_tokens": 1904, | |
| "output_tokens": 6754, | |
| "reasoning_output_tokens": 1904, | |
| "reasoning_reported_output_tokens": 6754, | |
| "reported_responses": { | |
| "cache_reported_input_tokens": 24, | |
| "cache_write_input_tokens": 24, | |
| "cache_write_reported_input_tokens": 24, | |
| "cached_input_tokens": 24, | |
| "input_tokens": 24, | |
| "output_tokens": 24, | |
| "reasoning_output_tokens": 24, | |
| "reasoning_reported_output_tokens": 24 | |
| }, | |
| "response_count": 24, | |
| "response_ids_complete": true, | |
| "schema": "rlebench/token-usage/1", | |
| "source": "Codex token_usage_record per response", | |
| "uncached_input_tokens": 35985, | |
| "unidentified_usage_records": 0 | |
| }, | |
| "kinex_usage": { | |
| "input_tokens": 1086944, | |
| "output_tokens": 7094, | |
| "cached_input_tokens": 980736, | |
| "cache_write_input_tokens": 0, | |
| "reasoning_output_tokens": 2071, | |
| "cache_reported_input_tokens": 1086944, | |
| "cache_write_reported_input_tokens": 1086944, | |
| "reasoning_reported_output_tokens": 7094, | |
| "schema": "rlebench/token-usage/1", | |
| "source": "Provider terminal response usage", | |
| "response_count": 40, | |
| "reported_responses": { | |
| "input_tokens": 40, | |
| "output_tokens": 40, | |
| "cached_input_tokens": 40, | |
| "cache_write_input_tokens": 40, | |
| "reasoning_output_tokens": 40, | |
| "cache_reported_input_tokens": 40, | |
| "cache_write_reported_input_tokens": 40, | |
| "reasoning_reported_output_tokens": 40 | |
| }, | |
| "known_cached_input_tokens": 980736, | |
| "known_input_tokens": 1086944, | |
| "known_output_tokens": 7094, | |
| "known_cache_write_input_tokens": 0, | |
| "known_reasoning_output_tokens": 2071, | |
| "cache_hit_rate": 0.9022875143521654, | |
| "uncached_input_tokens": 106208, | |
| "cost_usd": null, | |
| "response_ids_complete": true, | |
| "accounting": "reported-responses", | |
| "usage_source": "provider-raw", | |
| "request_attempts": 40, | |
| "rejected_attempts": 0, | |
| "unresolved_attempts": [], | |
| "audit_complete": true | |
| }, | |
| "codex_wall_time_s": 260.996992, | |
| "kinex_wall_time_s": 349.052826, | |
| "kinex_version": "0.10.3", | |
| "kinex_revision": "caac19a8a36272972f762e0f74cfe381e8500048", | |
| "same_roboenv_source": true | |
| }, | |
| { | |
| "task_key": "task02/08", | |
| "instruction": "put both the alphabet soup and the cream cheese box fully inside the basket", | |
| "instruction_policy": "modified", | |
| "codex_success": true, | |
| "kinex_success": true, | |
| "codex_steps": 1115, | |
| "kinex_steps": 926, | |
| "codex_usage": { | |
| "accounting": "reported-responses", | |
| "audit_complete": true, | |
| "cache_hit_rate": 0.9604073765886478, | |
| "cache_reported_input_tokens": 1406070, | |
| "cache_write_input_tokens": 0, | |
| "cache_write_reported_input_tokens": 1406070, | |
| "cached_input_tokens": 1350400, | |
| "completed_turns": 1, | |
| "cost_usd": null, | |
| "failed_turns": 0, | |
| "input_tokens": 1406070, | |
| "known_cache_write_input_tokens": 0, | |
| "known_cached_input_tokens": 1350400, | |
| "known_input_tokens": 1406070, | |
| "known_output_tokens": 9396, | |
| "known_reasoning_output_tokens": 4471, | |
| "output_tokens": 9396, | |
| "reasoning_output_tokens": 4471, | |
| "reasoning_reported_output_tokens": 9396, | |
| "reported_responses": { | |
| "cache_reported_input_tokens": 40, | |
| "cache_write_input_tokens": 40, | |
| "cache_write_reported_input_tokens": 40, | |
| "cached_input_tokens": 40, | |
| "input_tokens": 40, | |
| "output_tokens": 40, | |
| "reasoning_output_tokens": 40, | |
| "reasoning_reported_output_tokens": 40 | |
| }, | |
| "response_count": 40, | |
| "response_ids_complete": true, | |
| "schema": "rlebench/token-usage/1", | |
| "source": "Codex token_usage_record per response", | |
| "uncached_input_tokens": 55670, | |
| "unidentified_usage_records": 0 | |
| }, | |
| "kinex_usage": { | |
| "input_tokens": 1201086, | |
| "output_tokens": 6987, | |
| "cached_input_tokens": 1119616, | |
| "cache_write_input_tokens": 0, | |
| "reasoning_output_tokens": 2114, | |
| "cache_reported_input_tokens": 1201086, | |
| "cache_write_reported_input_tokens": 1201086, | |
| "reasoning_reported_output_tokens": 6987, | |
| "schema": "rlebench/token-usage/1", | |
| "source": "Provider terminal response usage", | |
| "response_count": 43, | |
| "reported_responses": { | |
| "input_tokens": 43, | |
| "output_tokens": 43, | |
| "cached_input_tokens": 43, | |
| "cache_write_input_tokens": 43, | |
| "reasoning_output_tokens": 43, | |
| "cache_reported_input_tokens": 43, | |
| "cache_write_reported_input_tokens": 43, | |
| "reasoning_reported_output_tokens": 43 | |
| }, | |
| "known_cached_input_tokens": 1119616, | |
| "known_input_tokens": 1201086, | |
| "known_output_tokens": 6987, | |
| "known_cache_write_input_tokens": 0, | |
| "known_reasoning_output_tokens": 2114, | |
| "cache_hit_rate": 0.9321697197369714, | |
| "uncached_input_tokens": 81470, | |
| "cost_usd": null, | |
| "response_ids_complete": true, | |
| "accounting": "reported-responses", | |
| "usage_source": "provider-raw", | |
| "request_attempts": 44, | |
| "rejected_attempts": 0, | |
| "unresolved_attempts": [], | |
| "audit_complete": true | |
| }, | |
| "codex_wall_time_s": 412.342365, | |
| "kinex_wall_time_s": 389.816733, | |
| "kinex_version": "0.10.3", | |
| "kinex_revision": "caac19a8a36272972f762e0f74cfe381e8500048", | |
| "same_roboenv_source": true | |
| }, | |
| { | |
| "task_key": "task02/09", | |
| "instruction": "put both moka pots on the stove and turn the stove on", | |
| "instruction_policy": "modified", | |
| "codex_success": true, | |
| "kinex_success": true, | |
| "codex_steps": 560, | |
| "kinex_steps": 890, | |
| "codex_usage": { | |
| "accounting": "reported-responses", | |
| "audit_complete": true, | |
| "cache_hit_rate": 0.9551717429852196, | |
| "cache_reported_input_tokens": 714527, | |
| "cache_write_input_tokens": 0, | |
| "cache_write_reported_input_tokens": 714527, | |
| "cached_input_tokens": 682496, | |
| "completed_turns": 1, | |
| "cost_usd": null, | |
| "failed_turns": 0, | |
| "input_tokens": 714527, | |
| "known_cache_write_input_tokens": 0, | |
| "known_cached_input_tokens": 682496, | |
| "known_input_tokens": 714527, | |
| "known_output_tokens": 5355, | |
| "known_reasoning_output_tokens": 1306, | |
| "output_tokens": 5355, | |
| "reasoning_output_tokens": 1306, | |
| "reasoning_reported_output_tokens": 5355, | |
| "reported_responses": { | |
| "cache_reported_input_tokens": 24, | |
| "cache_write_input_tokens": 24, | |
| "cache_write_reported_input_tokens": 24, | |
| "cached_input_tokens": 24, | |
| "input_tokens": 24, | |
| "output_tokens": 24, | |
| "reasoning_output_tokens": 24, | |
| "reasoning_reported_output_tokens": 24 | |
| }, | |
| "response_count": 24, | |
| "response_ids_complete": true, | |
| "schema": "rlebench/token-usage/1", | |
| "source": "Codex token_usage_record per response", | |
| "uncached_input_tokens": 32031, | |
| "unidentified_usage_records": 0 | |
| }, | |
| "kinex_usage": { | |
| "input_tokens": 1251289, | |
| "output_tokens": 7233, | |
| "cached_input_tokens": 1150848, | |
| "cache_write_input_tokens": 0, | |
| "reasoning_output_tokens": 2182, | |
| "cache_reported_input_tokens": 1251289, | |
| "cache_write_reported_input_tokens": 1251289, | |
| "reasoning_reported_output_tokens": 7233, | |
| "schema": "rlebench/token-usage/1", | |
| "source": "Provider terminal response usage", | |
| "response_count": 47, | |
| "reported_responses": { | |
| "input_tokens": 47, | |
| "output_tokens": 47, | |
| "cached_input_tokens": 47, | |
| "cache_write_input_tokens": 47, | |
| "reasoning_output_tokens": 47, | |
| "cache_reported_input_tokens": 47, | |
| "cache_write_reported_input_tokens": 47, | |
| "reasoning_reported_output_tokens": 47 | |
| }, | |
| "known_cached_input_tokens": 1150848, | |
| "known_input_tokens": 1251289, | |
| "known_output_tokens": 7233, | |
| "known_cache_write_input_tokens": 0, | |
| "known_reasoning_output_tokens": 2182, | |
| "cache_hit_rate": 0.9197299744503468, | |
| "uncached_input_tokens": 100441, | |
| "cost_usd": null, | |
| "response_ids_complete": true, | |
| "accounting": "reported-responses", | |
| "usage_source": "provider-raw", | |
| "request_attempts": 47, | |
| "rejected_attempts": 0, | |
| "unresolved_attempts": [], | |
| "audit_complete": true | |
| }, | |
| "codex_wall_time_s": 216.244925, | |
| "kinex_wall_time_s": 412.870858, | |
| "kinex_version": "0.10.3", | |
| "kinex_revision": "caac19a8a36272972f762e0f74cfe381e8500048", | |
| "same_roboenv_source": true | |
| }, | |
| { | |
| "task_key": "task02/10", | |
| "instruction": "put the yellow and white mug in the microwave and close it", | |
| "instruction_policy": "original_native", | |
| "codex_success": true, | |
| "kinex_success": true, | |
| "codex_steps": 1210, | |
| "kinex_steps": 1950, | |
| "codex_usage": { | |
| "accounting": "reported-responses", | |
| "audit_complete": true, | |
| "cache_hit_rate": 0.9540138123860797, | |
| "cache_reported_input_tokens": 1349079, | |
| "cache_write_input_tokens": 0, | |
| "cache_write_reported_input_tokens": 1349079, | |
| "cached_input_tokens": 1287040, | |
| "completed_turns": 1, | |
| "cost_usd": null, | |
| "failed_turns": 0, | |
| "input_tokens": 1349079, | |
| "known_cache_write_input_tokens": 0, | |
| "known_cached_input_tokens": 1287040, | |
| "known_input_tokens": 1349079, | |
| "known_output_tokens": 10556, | |
| "known_reasoning_output_tokens": 4068, | |
| "output_tokens": 10556, | |
| "reasoning_output_tokens": 4068, | |
| "reasoning_reported_output_tokens": 10556, | |
| "reported_responses": { | |
| "cache_reported_input_tokens": 39, | |
| "cache_write_input_tokens": 39, | |
| "cache_write_reported_input_tokens": 39, | |
| "cached_input_tokens": 39, | |
| "input_tokens": 39, | |
| "output_tokens": 39, | |
| "reasoning_output_tokens": 39, | |
| "reasoning_reported_output_tokens": 39 | |
| }, | |
| "response_count": 39, | |
| "response_ids_complete": true, | |
| "schema": "rlebench/token-usage/1", | |
| "source": "Codex token_usage_record per response", | |
| "uncached_input_tokens": 62039, | |
| "unidentified_usage_records": 0 | |
| }, | |
| "kinex_usage": { | |
| "input_tokens": 3336633, | |
| "output_tokens": 12622, | |
| "cached_input_tokens": 3199872, | |
| "cache_write_input_tokens": 0, | |
| "reasoning_output_tokens": 3738, | |
| "cache_reported_input_tokens": 3336633, | |
| "cache_write_reported_input_tokens": 3336633, | |
| "reasoning_reported_output_tokens": 12622, | |
| "schema": "rlebench/token-usage/1", | |
| "source": "Provider terminal response usage", | |
| "response_count": 82, | |
| "reported_responses": { | |
| "input_tokens": 82, | |
| "output_tokens": 82, | |
| "cached_input_tokens": 82, | |
| "cache_write_input_tokens": 82, | |
| "reasoning_output_tokens": 82, | |
| "cache_reported_input_tokens": 82, | |
| "cache_write_reported_input_tokens": 82, | |
| "reasoning_reported_output_tokens": 82 | |
| }, | |
| "known_cached_input_tokens": 3199872, | |
| "known_input_tokens": 3336633, | |
| "known_output_tokens": 12622, | |
| "known_cache_write_input_tokens": 0, | |
| "known_reasoning_output_tokens": 3738, | |
| "cache_hit_rate": 0.9590122737502147, | |
| "uncached_input_tokens": 136761, | |
| "cost_usd": null, | |
| "response_ids_complete": true, | |
| "accounting": "reported-responses", | |
| "usage_source": "provider-raw", | |
| "request_attempts": 82, | |
| "rejected_attempts": 0, | |
| "unresolved_attempts": [], | |
| "audit_complete": true | |
| }, | |
| "codex_wall_time_s": 399.959252, | |
| "kinex_wall_time_s": 717.324727, | |
| "kinex_version": "0.10.3", | |
| "kinex_revision": "caac19a8a36272972f762e0f74cfe381e8500048", | |
| "same_roboenv_source": true | |
| } | |
| ], | |
| "preserved_robodojo_comparison": { | |
| "baseline_revision": "1a7f1c9f178ba7aeb659e5b527a77978e9136072", | |
| "summary": { | |
| "completed": 42, | |
| "codex_successes": 31, | |
| "codex_success_rate_completed_subset": 0.7380952380952381, | |
| "kinex_successes_same_subset": 35, | |
| "kinex_successes_all42": 35, | |
| "all42_comparison_complete": true | |
| }, | |
| "tasks": [ | |
| { | |
| "task_key": "task04/01", | |
| "native_id": "robodojo/make-toast", | |
| "status": "completed", | |
| "codex_success": false, | |
| "kinex_success": false, | |
| "kinex_version": "0.10.3" | |
| }, | |
| { | |
| "task_key": "task04/02", | |
| "native_id": "robodojo/classify-objects-by-language", | |
| "status": "completed", | |
| "codex_success": true, | |
| "kinex_success": true, | |
| "kinex_version": "0.10.0" | |
| }, | |
| { | |
| "task_key": "task04/03", | |
| "native_id": "robodojo/store-laptop-and-headphones", | |
| "status": "completed", | |
| "codex_success": false, | |
| "kinex_success": false, | |
| "kinex_version": "0.10.3" | |
| }, | |
| { | |
| "task_key": "task04/04", | |
| "native_id": "robodojo/cover-blocks", | |
| "status": "completed", | |
| "codex_success": true, | |
| "kinex_success": true, | |
| "kinex_version": "0.10.2" | |
| }, | |
| { | |
| "task_key": "task04/05", | |
| "native_id": "robodojo/play-xylophone", | |
| "status": "completed", | |
| "codex_success": true, | |
| "kinex_success": true, | |
| "kinex_version": "0.10.0" | |
| }, | |
| { | |
| "task_key": "task04/06", | |
| "native_id": "robodojo/store-tools-in-toolbox", | |
| "status": "completed", | |
| "codex_success": false, | |
| "kinex_success": false, | |
| "kinex_version": "0.10.3" | |
| }, | |
| { | |
| "task_key": "task04/07", | |
| "native_id": "robodojo/insert-tubes", | |
| "status": "completed", | |
| "codex_success": true, | |
| "kinex_success": true, | |
| "kinex_version": "0.10.3" | |
| }, | |
| { | |
| "task_key": "task04/08", | |
| "native_id": "robodojo/deposit-coin", | |
| "status": "completed", | |
| "codex_success": true, | |
| "kinex_success": true, | |
| "kinex_version": "0.10.2" | |
| }, | |
| { | |
| "task_key": "task04/09", | |
| "native_id": "robodojo/fasten-screws", | |
| "status": "completed", | |
| "codex_success": true, | |
| "kinex_success": true, | |
| "kinex_version": "0.10.3" | |
| }, | |
| { | |
| "task_key": "task04/10", | |
| "native_id": "robodojo/play-stacking-toy", | |
| "status": "completed", | |
| "codex_success": true, | |
| "kinex_success": true, | |
| "kinex_version": "0.10.3" | |
| }, | |
| { | |
| "task_key": "task04/11", | |
| "native_id": "robodojo/align-blocks", | |
| "status": "completed", | |
| "codex_success": true, | |
| "kinex_success": true, | |
| "kinex_version": "0.10.0" | |
| }, | |
| { | |
| "task_key": "task04/12", | |
| "native_id": "robodojo/arrange-largest-number", | |
| "status": "completed", | |
| "codex_success": true, | |
| "kinex_success": true, | |
| "kinex_version": "0.10.3" | |
| }, | |
| { | |
| "task_key": "task04/14", | |
| "native_id": "robodojo/build-tower", | |
| "status": "completed", | |
| "codex_success": true, | |
| "kinex_success": true, | |
| "kinex_version": "0.10.3" | |
| }, | |
| { | |
| "task_key": "task04/15", | |
| "native_id": "robodojo/classify-objects", | |
| "status": "completed", | |
| "codex_success": true, | |
| "kinex_success": true, | |
| "kinex_version": "0.10.2" | |
| }, | |
| { | |
| "task_key": "task04/16", | |
| "native_id": "robodojo/fill-egg-holder", | |
| "status": "completed", | |
| "codex_success": true, | |
| "kinex_success": false, | |
| "kinex_version": "0.10.3" | |
| }, | |
| { | |
| "task_key": "task04/17", | |
| "native_id": "robodojo/fill-pen-holder", | |
| "status": "completed", | |
| "codex_success": false, | |
| "kinex_success": false, | |
| "kinex_version": "0.10.3" | |
| }, | |
| { | |
| "task_key": "task04/18", | |
| "native_id": "robodojo/fold-clothes", | |
| "status": "completed", | |
| "codex_success": true, | |
| "kinex_success": true, | |
| "kinex_version": "0.10.3" | |
| }, | |
| { | |
| "task_key": "task04/20", | |
| "native_id": "robodojo/general-pickup", | |
| "status": "completed", | |
| "codex_success": true, | |
| "kinex_success": true, | |
| "kinex_version": "0.10.0" | |
| }, | |
| { | |
| "task_key": "task04/21", | |
| "native_id": "robodojo/hang-mugs", | |
| "status": "completed", | |
| "codex_success": true, | |
| "kinex_success": true, | |
| "kinex_version": "0.10.3" | |
| }, | |
| { | |
| "task_key": "task04/23", | |
| "native_id": "robodojo/imitate-sorting-sequence", | |
| "status": "completed", | |
| "codex_success": true, | |
| "kinex_success": true, | |
| "kinex_version": "0.10.2" | |
| }, | |
| { | |
| "task_key": "task04/24", | |
| "native_id": "robodojo/insert-key", | |
| "status": "completed", | |
| "codex_success": true, | |
| "kinex_success": true, | |
| "kinex_version": "0.10.2" | |
| }, | |
| { | |
| "task_key": "task04/25", | |
| "native_id": "robodojo/make-kong", | |
| "status": "completed", | |
| "codex_success": false, | |
| "kinex_success": true, | |
| "kinex_version": "0.10.3" | |
| }, | |
| { | |
| "task_key": "task04/27", | |
| "native_id": "robodojo/match-and-pick-from-conveyor", | |
| "status": "completed", | |
| "codex_success": true, | |
| "kinex_success": true, | |
| "kinex_version": "0.10.3" | |
| }, | |
| { | |
| "task_key": "task04/28", | |
| "native_id": "robodojo/organize-table", | |
| "status": "completed", | |
| "codex_success": false, | |
| "kinex_success": true, | |
| "kinex_version": "0.10.3" | |
| }, | |
| { | |
| "task_key": "task04/29", | |
| "native_id": "robodojo/pack-objects-into-box", | |
| "status": "completed", | |
| "codex_success": true, | |
| "kinex_success": true, | |
| "kinex_version": "0.10.2" | |
| }, | |
| { | |
| "task_key": "task04/31", | |
| "native_id": "robodojo/pick-from-conveyor-by-image", | |
| "status": "completed", | |
| "codex_success": false, | |
| "kinex_success": false, | |
| "kinex_version": "0.10.3" | |
| }, | |
| { | |
| "task_key": "task04/32", | |
| "native_id": "robodojo/play-tic-tac-toe", | |
| "status": "completed", | |
| "codex_success": true, | |
| "kinex_success": true, | |
| "kinex_version": "0.10.3" | |
| }, | |
| { | |
| "task_key": "task04/33", | |
| "native_id": "robodojo/plug-in-charger", | |
| "status": "completed", | |
| "codex_success": true, | |
| "kinex_success": true, | |
| "kinex_version": "0.10.3" | |
| }, | |
| { | |
| "task_key": "task04/34", | |
| "native_id": "robodojo/pour-balls-into-vase", | |
| "status": "completed", | |
| "codex_success": true, | |
| "kinex_success": true, | |
| "kinex_version": "0.10.3" | |
| }, | |
| { | |
| "task_key": "task04/35", | |
| "native_id": "robodojo/pour-by-language", | |
| "status": "completed", | |
| "codex_success": false, | |
| "kinex_success": true, | |
| "kinex_version": "0.10.0" | |
| }, | |
| { | |
| "task_key": "task04/36", | |
| "native_id": "robodojo/pour-liquid-into-cup", | |
| "status": "completed", | |
| "codex_success": true, | |
| "kinex_success": true, | |
| "kinex_version": "0.10.3" | |
| }, | |
| { | |
| "task_key": "task04/38", | |
| "native_id": "robodojo/press-by-number", | |
| "status": "completed", | |
| "codex_success": true, | |
| "kinex_success": true, | |
| "kinex_version": "0.10.3" | |
| }, | |
| { | |
| "task_key": "task04/39", | |
| "native_id": "robodojo/push-t", | |
| "status": "completed", | |
| "codex_success": false, | |
| "kinex_success": true, | |
| "kinex_version": "0.10.3" | |
| }, | |
| { | |
| "task_key": "task04/41", | |
| "native_id": "robodojo/put-bottles-into-dustbin", | |
| "status": "completed", | |
| "codex_success": true, | |
| "kinex_success": true, | |
| "kinex_version": "0.10.2" | |
| }, | |
| { | |
| "task_key": "task04/42", | |
| "native_id": "robodojo/solve-equation", | |
| "status": "completed", | |
| "codex_success": true, | |
| "kinex_success": true, | |
| "kinex_version": "0.10.0" | |
| }, | |
| { | |
| "task_key": "task04/43", | |
| "native_id": "robodojo/sort-nesting-dolls-by-size", | |
| "status": "completed", | |
| "codex_success": true, | |
| "kinex_success": true, | |
| "kinex_version": "0.10.3" | |
| }, | |
| { | |
| "task_key": "task04/45", | |
| "native_id": "robodojo/stack-blocks", | |
| "status": "completed", | |
| "codex_success": true, | |
| "kinex_success": true, | |
| "kinex_version": "0.10.2" | |
| }, | |
| { | |
| "task_key": "task04/46", | |
| "native_id": "robodojo/stack-blocks-by-language", | |
| "status": "completed", | |
| "codex_success": true, | |
| "kinex_success": true, | |
| "kinex_version": "0.10.0" | |
| }, | |
| { | |
| "task_key": "task04/48", | |
| "native_id": "robodojo/stack-bowls", | |
| "status": "completed", | |
| "codex_success": true, | |
| "kinex_success": true, | |
| "kinex_version": "0.10.2" | |
| }, | |
| { | |
| "task_key": "task04/51", | |
| "native_id": "robodojo/swap-t", | |
| "status": "completed", | |
| "codex_success": true, | |
| "kinex_success": true, | |
| "kinex_version": "0.10.2" | |
| }, | |
| { | |
| "task_key": "task04/52", | |
| "native_id": "robodojo/swap-blocks", | |
| "status": "completed", | |
| "codex_success": false, | |
| "kinex_success": false, | |
| "kinex_version": "0.10.3" | |
| }, | |
| { | |
| "task_key": "task04/53", | |
| "native_id": "robodojo/sweep-blocks", | |
| "status": "completed", | |
| "codex_success": false, | |
| "kinex_success": true, | |
| "kinex_version": "0.10.2" | |
| } | |
| ], | |
| "limitations": [ | |
| "Historical Kinex baseline mixes three runtime versions.", | |
| "24 tasks share RoboEnv source/build; 18 use an older build.", | |
| "The first six completed Codex tasks used original request defaults; later attempts use 50 retries." | |
| ] | |
| }, | |
| "limitations": [ | |
| "Single seed and one native episode per task; these are observed success rates, not population estimates.", | |
| "Wall time is operational latency under different concurrency, not a controlled speed benchmark; Kinex used up to five concurrent tasks and the new Codex dispatcher admits up to twenty.", | |
| "LIBERO Kinex uses caac19a. The retained RoboCasa Kinex reference uses a1eaece; the two intervening Kinex commits change session timing and footer display.", | |
| "The ordinary Codex baseline uses its native exec tools and the public RoboEnv SDK/CLI. Kinex uses its robotics tools.", | |
| "RoboDojo retains its previously published protocol and historical source differences." | |
| ], | |
| "robotwin": { | |
| "reference_repo": "RLE-Bench/kinex-v0.10.0-benchmark", | |
| "reference_heads": { | |
| "dataset": { | |
| "sha": "b8b577c704c297f0bcf10f0bb195cb14576db4b9", | |
| "files": 9026, | |
| "private": false, | |
| "host": null, | |
| "bytes": 5217283103 | |
| }, | |
| "space": { | |
| "sha": "f59eb8b6114a517c9629a3f7ae10cbd1c7ca19c2", | |
| "files": 528, | |
| "private": false, | |
| "host": "https://rle-bench-kinex-v0-10-0-benchmark.static.hf.space", | |
| "bytes": 125334443 | |
| } | |
| }, | |
| "tasks": [ | |
| { | |
| "task_key": "task03/01", | |
| "codex_success": true, | |
| "kinex_success": true, | |
| "codex_steps": 803, | |
| "kinex_steps": 1940, | |
| "same_instruction": false, | |
| "same_instruction_policy": false, | |
| "same_roboenv_source": false, | |
| "codex_usage": { | |
| "accounting": "reported-responses", | |
| "audit_complete": true, | |
| "cache_hit_rate": 0.9529955328334399, | |
| "cache_reported_input_tokens": 640003, | |
| "cache_write_input_tokens": 0, | |
| "cache_write_reported_input_tokens": 640003, | |
| "cached_input_tokens": 609920, | |
| "completed_turns": 1, | |
| "cost_usd": null, | |
| "failed_turns": 0, | |
| "input_tokens": 640003, | |
| "known_cache_write_input_tokens": 0, | |
| "known_cached_input_tokens": 609920, | |
| "known_input_tokens": 640003, | |
| "known_output_tokens": 6267, | |
| "known_reasoning_output_tokens": 1323, | |
| "output_tokens": 6267, | |
| "reasoning_output_tokens": 1323, | |
| "reasoning_reported_output_tokens": 6267, | |
| "reported_responses": { | |
| "cache_reported_input_tokens": 22, | |
| "cache_write_input_tokens": 22, | |
| "cache_write_reported_input_tokens": 22, | |
| "cached_input_tokens": 22, | |
| "input_tokens": 22, | |
| "output_tokens": 22, | |
| "reasoning_output_tokens": 22, | |
| "reasoning_reported_output_tokens": 22 | |
| }, | |
| "response_count": 22, | |
| "response_ids_complete": true, | |
| "schema": "rlebench/token-usage/1", | |
| "source": "Codex token_usage_record per response", | |
| "uncached_input_tokens": 30083, | |
| "unidentified_usage_records": 0 | |
| }, | |
| "kinex_usage": { | |
| "input_tokens": 1875971, | |
| "output_tokens": 11332, | |
| "cached_input_tokens": 1778560, | |
| "cache_write_input_tokens": 0, | |
| "reasoning_output_tokens": 3668, | |
| "cache_reported_input_tokens": 1875971, | |
| "cache_write_reported_input_tokens": 1875971, | |
| "reasoning_reported_output_tokens": 11332, | |
| "schema": "rlebench/token-usage/1", | |
| "source": "Provider terminal response usage", | |
| "response_count": 56, | |
| "reported_responses": { | |
| "input_tokens": 56, | |
| "output_tokens": 56, | |
| "cached_input_tokens": 56, | |
| "cache_write_input_tokens": 56, | |
| "reasoning_output_tokens": 56, | |
| "cache_reported_input_tokens": 56, | |
| "cache_write_reported_input_tokens": 56, | |
| "reasoning_reported_output_tokens": 56 | |
| }, | |
| "known_cached_input_tokens": 1778560, | |
| "known_input_tokens": 1875971, | |
| "known_output_tokens": 11332, | |
| "known_cache_write_input_tokens": 0, | |
| "known_reasoning_output_tokens": 3668, | |
| "cache_hit_rate": 0.9480743572262045, | |
| "uncached_input_tokens": 97411, | |
| "cost_usd": null, | |
| "response_ids_complete": true, | |
| "usage_source": "provider-raw", | |
| "request_attempts": 56, | |
| "rejected_attempts": 0, | |
| "unresolved_attempts": [], | |
| "audit_complete": true | |
| } | |
| }, | |
| { | |
| "task_key": "task03/02", | |
| "codex_success": true, | |
| "kinex_success": true, | |
| "codex_steps": 776, | |
| "kinex_steps": 836, | |
| "same_instruction": true, | |
| "same_instruction_policy": true, | |
| "same_roboenv_source": false, | |
| "codex_usage": { | |
| "accounting": "reported-responses", | |
| "audit_complete": true, | |
| "cache_hit_rate": 0.9590275594676726, | |
| "cache_reported_input_tokens": 1106988, | |
| "cache_write_input_tokens": 0, | |
| "cache_write_reported_input_tokens": 1106988, | |
| "cached_input_tokens": 1061632, | |
| "completed_turns": 1, | |
| "cost_usd": null, | |
| "failed_turns": 0, | |
| "input_tokens": 1106988, | |
| "known_cache_write_input_tokens": 0, | |
| "known_cached_input_tokens": 1061632, | |
| "known_input_tokens": 1106988, | |
| "known_output_tokens": 9915, | |
| "known_reasoning_output_tokens": 3875, | |
| "output_tokens": 9915, | |
| "reasoning_output_tokens": 3875, | |
| "reasoning_reported_output_tokens": 9915, | |
| "reported_responses": { | |
| "cache_reported_input_tokens": 31, | |
| "cache_write_input_tokens": 31, | |
| "cache_write_reported_input_tokens": 31, | |
| "cached_input_tokens": 31, | |
| "input_tokens": 31, | |
| "output_tokens": 31, | |
| "reasoning_output_tokens": 31, | |
| "reasoning_reported_output_tokens": 31 | |
| }, | |
| "response_count": 31, | |
| "response_ids_complete": true, | |
| "schema": "rlebench/token-usage/1", | |
| "source": "Codex token_usage_record per response", | |
| "uncached_input_tokens": 45356, | |
| "unidentified_usage_records": 0 | |
| }, | |
| "kinex_usage": { | |
| "input_tokens": 1396364, | |
| "output_tokens": 8842, | |
| "cached_input_tokens": 1312384, | |
| "cache_write_input_tokens": 0, | |
| "reasoning_output_tokens": 3143, | |
| "cache_reported_input_tokens": 1396364, | |
| "cache_write_reported_input_tokens": 1396364, | |
| "reasoning_reported_output_tokens": 8842, | |
| "schema": "rlebench/token-usage/1", | |
| "source": "Provider terminal response usage", | |
| "response_count": 44, | |
| "reported_responses": { | |
| "input_tokens": 44, | |
| "output_tokens": 44, | |
| "cached_input_tokens": 44, | |
| "cache_write_input_tokens": 44, | |
| "reasoning_output_tokens": 44, | |
| "cache_reported_input_tokens": 44, | |
| "cache_write_reported_input_tokens": 44, | |
| "reasoning_reported_output_tokens": 44 | |
| }, | |
| "known_cached_input_tokens": 1312384, | |
| "known_input_tokens": 1396364, | |
| "known_output_tokens": 8842, | |
| "known_cache_write_input_tokens": 0, | |
| "known_reasoning_output_tokens": 3143, | |
| "cache_hit_rate": 0.9398580885786227, | |
| "uncached_input_tokens": 83980, | |
| "cost_usd": null, | |
| "response_ids_complete": true, | |
| "usage_source": "provider-raw", | |
| "request_attempts": 44, | |
| "rejected_attempts": 0, | |
| "unresolved_attempts": [], | |
| "audit_complete": true, | |
| "accounting": "reported-responses" | |
| } | |
| }, | |
| { | |
| "task_key": "task03/03", | |
| "codex_success": false, | |
| "kinex_success": true, | |
| "codex_steps": 2790, | |
| "kinex_steps": 2064, | |
| "same_instruction": true, | |
| "same_instruction_policy": true, | |
| "same_roboenv_source": false, | |
| "codex_usage": { | |
| "accounting": "reported-responses", | |
| "audit_complete": true, | |
| "cache_hit_rate": 0.9849180997132441, | |
| "cache_reported_input_tokens": 3977947, | |
| "cache_write_input_tokens": 0, | |
| "cache_write_reported_input_tokens": 3977947, | |
| "cached_input_tokens": 3917952, | |
| "completed_turns": 1, | |
| "cost_usd": null, | |
| "failed_turns": 0, | |
| "input_tokens": 3977947, | |
| "known_cache_write_input_tokens": 0, | |
| "known_cached_input_tokens": 3917952, | |
| "known_input_tokens": 3977947, | |
| "known_output_tokens": 19047, | |
| "known_reasoning_output_tokens": 8202, | |
| "output_tokens": 19047, | |
| "reasoning_output_tokens": 8202, | |
| "reasoning_reported_output_tokens": 19047, | |
| "reported_responses": { | |
| "cache_reported_input_tokens": 97, | |
| "cache_write_input_tokens": 97, | |
| "cache_write_reported_input_tokens": 97, | |
| "cached_input_tokens": 97, | |
| "input_tokens": 97, | |
| "output_tokens": 97, | |
| "reasoning_output_tokens": 97, | |
| "reasoning_reported_output_tokens": 97 | |
| }, | |
| "response_count": 97, | |
| "response_ids_complete": true, | |
| "schema": "rlebench/token-usage/1", | |
| "source": "Codex token_usage_record per response", | |
| "uncached_input_tokens": 59995, | |
| "unidentified_usage_records": 0 | |
| }, | |
| "kinex_usage": { | |
| "input_tokens": 1997820, | |
| "output_tokens": 6541, | |
| "cached_input_tokens": 1914368, | |
| "cache_write_input_tokens": 0, | |
| "reasoning_output_tokens": 1414, | |
| "cache_reported_input_tokens": 1997820, | |
| "cache_write_reported_input_tokens": 1997820, | |
| "reasoning_reported_output_tokens": 6541, | |
| "schema": "rlebench/token-usage/1", | |
| "source": "Provider terminal response usage", | |
| "response_count": 73, | |
| "reported_responses": { | |
| "input_tokens": 73, | |
| "output_tokens": 73, | |
| "cached_input_tokens": 73, | |
| "cache_write_input_tokens": 73, | |
| "reasoning_output_tokens": 73, | |
| "cache_reported_input_tokens": 73, | |
| "cache_write_reported_input_tokens": 73, | |
| "reasoning_reported_output_tokens": 73 | |
| }, | |
| "known_cached_input_tokens": 1914368, | |
| "known_input_tokens": 1997820, | |
| "known_output_tokens": 6541, | |
| "known_cache_write_input_tokens": 0, | |
| "known_reasoning_output_tokens": 1414, | |
| "cache_hit_rate": 0.958228469031244, | |
| "uncached_input_tokens": 83452, | |
| "cost_usd": null, | |
| "response_ids_complete": true, | |
| "usage_source": "provider-raw", | |
| "request_attempts": 73, | |
| "rejected_attempts": 0, | |
| "unresolved_attempts": [], | |
| "audit_complete": true | |
| } | |
| }, | |
| { | |
| "task_key": "task03/04", | |
| "codex_success": false, | |
| "kinex_success": false, | |
| "codex_steps": 1277, | |
| "kinex_steps": 1634, | |
| "same_instruction": true, | |
| "same_instruction_policy": true, | |
| "same_roboenv_source": true, | |
| "codex_usage": { | |
| "accounting": "reported-responses", | |
| "audit_complete": true, | |
| "cache_hit_rate": 0.9479087578186759, | |
| "cache_reported_input_tokens": 1663485, | |
| "cache_write_input_tokens": 0, | |
| "cache_write_reported_input_tokens": 1663485, | |
| "cached_input_tokens": 1576832, | |
| "completed_turns": 1, | |
| "cost_usd": null, | |
| "failed_turns": 0, | |
| "input_tokens": 1663485, | |
| "known_cache_write_input_tokens": 0, | |
| "known_cached_input_tokens": 1576832, | |
| "known_input_tokens": 1663485, | |
| "known_output_tokens": 11054, | |
| "known_reasoning_output_tokens": 3714, | |
| "output_tokens": 11054, | |
| "reasoning_output_tokens": 3714, | |
| "reasoning_reported_output_tokens": 11054, | |
| "reported_responses": { | |
| "cache_reported_input_tokens": 46, | |
| "cache_write_input_tokens": 46, | |
| "cache_write_reported_input_tokens": 46, | |
| "cached_input_tokens": 46, | |
| "input_tokens": 46, | |
| "output_tokens": 46, | |
| "reasoning_output_tokens": 46, | |
| "reasoning_reported_output_tokens": 46 | |
| }, | |
| "response_count": 46, | |
| "response_ids_complete": true, | |
| "schema": "rlebench/token-usage/1", | |
| "source": "Codex token_usage_record per response", | |
| "uncached_input_tokens": 86653, | |
| "unidentified_usage_records": 0 | |
| }, | |
| "kinex_usage": { | |
| "input_tokens": 2492378, | |
| "output_tokens": 10795, | |
| "cached_input_tokens": 2383360, | |
| "cache_write_input_tokens": 0, | |
| "reasoning_output_tokens": 4000, | |
| "cache_reported_input_tokens": 2492378, | |
| "cache_write_reported_input_tokens": 2492378, | |
| "reasoning_reported_output_tokens": 10795, | |
| "schema": "rlebench/token-usage/1", | |
| "source": "Provider terminal response usage", | |
| "response_count": 66, | |
| "reported_responses": { | |
| "input_tokens": 66, | |
| "output_tokens": 66, | |
| "cached_input_tokens": 66, | |
| "cache_write_input_tokens": 66, | |
| "reasoning_output_tokens": 66, | |
| "cache_reported_input_tokens": 66, | |
| "cache_write_reported_input_tokens": 66, | |
| "reasoning_reported_output_tokens": 66 | |
| }, | |
| "known_cached_input_tokens": 2383360, | |
| "known_input_tokens": 2492378, | |
| "known_output_tokens": 10795, | |
| "known_cache_write_input_tokens": 0, | |
| "known_reasoning_output_tokens": 4000, | |
| "cache_hit_rate": 0.9562594437922337, | |
| "uncached_input_tokens": 109018, | |
| "cost_usd": null, | |
| "response_ids_complete": true, | |
| "accounting": "reported-responses", | |
| "usage_source": "provider-raw", | |
| "request_attempts": 66, | |
| "rejected_attempts": 0, | |
| "unresolved_attempts": [], | |
| "audit_complete": true | |
| } | |
| }, | |
| { | |
| "task_key": "task03/05", | |
| "codex_success": false, | |
| "kinex_success": true, | |
| "codex_steps": 3263, | |
| "kinex_steps": 2172, | |
| "same_instruction": true, | |
| "same_instruction_policy": true, | |
| "same_roboenv_source": true, | |
| "codex_usage": { | |
| "accounting": "reported-responses", | |
| "audit_complete": true, | |
| "cache_hit_rate": 0.9794733490044869, | |
| "cache_reported_input_tokens": 3072737, | |
| "cache_write_input_tokens": 0, | |
| "cache_write_reported_input_tokens": 3072737, | |
| "cached_input_tokens": 3009664, | |
| "completed_turns": 1, | |
| "cost_usd": null, | |
| "failed_turns": 0, | |
| "input_tokens": 3072737, | |
| "known_cache_write_input_tokens": 0, | |
| "known_cached_input_tokens": 3009664, | |
| "known_input_tokens": 3072737, | |
| "known_output_tokens": 20636, | |
| "known_reasoning_output_tokens": 8283, | |
| "output_tokens": 20636, | |
| "reasoning_output_tokens": 8283, | |
| "reasoning_reported_output_tokens": 20636, | |
| "reported_responses": { | |
| "cache_reported_input_tokens": 71, | |
| "cache_write_input_tokens": 71, | |
| "cache_write_reported_input_tokens": 71, | |
| "cached_input_tokens": 71, | |
| "input_tokens": 71, | |
| "output_tokens": 71, | |
| "reasoning_output_tokens": 71, | |
| "reasoning_reported_output_tokens": 71 | |
| }, | |
| "response_count": 71, | |
| "response_ids_complete": true, | |
| "schema": "rlebench/token-usage/1", | |
| "source": "Codex token_usage_record per response", | |
| "uncached_input_tokens": 63073, | |
| "unidentified_usage_records": 0 | |
| }, | |
| "kinex_usage": { | |
| "input_tokens": 2470735, | |
| "output_tokens": 13511, | |
| "cached_input_tokens": 2363776, | |
| "cache_write_input_tokens": 0, | |
| "reasoning_output_tokens": 4117, | |
| "cache_reported_input_tokens": 2470735, | |
| "cache_write_reported_input_tokens": 2470735, | |
| "reasoning_reported_output_tokens": 13511, | |
| "schema": "rlebench/token-usage/1", | |
| "source": "Provider terminal response usage", | |
| "response_count": 70, | |
| "reported_responses": { | |
| "input_tokens": 70, | |
| "output_tokens": 70, | |
| "cached_input_tokens": 70, | |
| "cache_write_input_tokens": 70, | |
| "reasoning_output_tokens": 70, | |
| "cache_reported_input_tokens": 70, | |
| "cache_write_reported_input_tokens": 70, | |
| "reasoning_reported_output_tokens": 70 | |
| }, | |
| "known_cached_input_tokens": 2363776, | |
| "known_input_tokens": 2470735, | |
| "known_output_tokens": 13511, | |
| "known_cache_write_input_tokens": 0, | |
| "known_reasoning_output_tokens": 4117, | |
| "cache_hit_rate": 0.956709643081917, | |
| "uncached_input_tokens": 106959, | |
| "cost_usd": null, | |
| "response_ids_complete": true, | |
| "accounting": "reported-responses", | |
| "usage_source": "provider-raw", | |
| "request_attempts": 70, | |
| "rejected_attempts": 0, | |
| "unresolved_attempts": [], | |
| "audit_complete": true | |
| } | |
| }, | |
| { | |
| "task_key": "task03/06", | |
| "codex_success": true, | |
| "kinex_success": true, | |
| "codex_steps": 905, | |
| "kinex_steps": 2602, | |
| "same_instruction": true, | |
| "same_instruction_policy": true, | |
| "same_roboenv_source": false, | |
| "codex_usage": { | |
| "accounting": "reported-responses", | |
| "audit_complete": true, | |
| "cache_hit_rate": 0.961291401007713, | |
| "cache_reported_input_tokens": 921294, | |
| "cache_write_input_tokens": 0, | |
| "cache_write_reported_input_tokens": 921294, | |
| "cached_input_tokens": 885632, | |
| "completed_turns": 1, | |
| "cost_usd": null, | |
| "failed_turns": 0, | |
| "input_tokens": 921294, | |
| "known_cache_write_input_tokens": 0, | |
| "known_cached_input_tokens": 885632, | |
| "known_input_tokens": 921294, | |
| "known_output_tokens": 6531, | |
| "known_reasoning_output_tokens": 1389, | |
| "output_tokens": 6531, | |
| "reasoning_output_tokens": 1389, | |
| "reasoning_reported_output_tokens": 6531, | |
| "reported_responses": { | |
| "cache_reported_input_tokens": 28, | |
| "cache_write_input_tokens": 28, | |
| "cache_write_reported_input_tokens": 28, | |
| "cached_input_tokens": 28, | |
| "input_tokens": 28, | |
| "output_tokens": 28, | |
| "reasoning_output_tokens": 28, | |
| "reasoning_reported_output_tokens": 28 | |
| }, | |
| "response_count": 28, | |
| "response_ids_complete": true, | |
| "schema": "rlebench/token-usage/1", | |
| "source": "Codex token_usage_record per response", | |
| "uncached_input_tokens": 35662, | |
| "unidentified_usage_records": 0 | |
| }, | |
| "kinex_usage": { | |
| "input_tokens": 1123532, | |
| "output_tokens": 9317, | |
| "cached_input_tokens": 1035136, | |
| "cache_write_input_tokens": 0, | |
| "reasoning_output_tokens": 2828, | |
| "cache_reported_input_tokens": 1123532, | |
| "cache_write_reported_input_tokens": 1123532, | |
| "reasoning_reported_output_tokens": 9317, | |
| "schema": "rlebench/token-usage/1", | |
| "source": "Provider terminal response usage", | |
| "response_count": 34, | |
| "reported_responses": { | |
| "input_tokens": 34, | |
| "output_tokens": 34, | |
| "cached_input_tokens": 34, | |
| "cache_write_input_tokens": 34, | |
| "reasoning_output_tokens": 34, | |
| "cache_reported_input_tokens": 34, | |
| "cache_write_reported_input_tokens": 34, | |
| "reasoning_reported_output_tokens": 34 | |
| }, | |
| "known_cached_input_tokens": 1035136, | |
| "known_input_tokens": 1123532, | |
| "known_output_tokens": 9317, | |
| "known_cache_write_input_tokens": 0, | |
| "known_reasoning_output_tokens": 2828, | |
| "cache_hit_rate": 0.921323113182357, | |
| "uncached_input_tokens": 88396, | |
| "cost_usd": null, | |
| "response_ids_complete": true, | |
| "usage_source": "provider-raw", | |
| "request_attempts": 34, | |
| "rejected_attempts": 0, | |
| "unresolved_attempts": [], | |
| "audit_complete": true | |
| } | |
| }, | |
| { | |
| "task_key": "task03/07", | |
| "codex_success": true, | |
| "kinex_success": true, | |
| "codex_steps": 870, | |
| "kinex_steps": 3537, | |
| "same_instruction": true, | |
| "same_instruction_policy": true, | |
| "same_roboenv_source": false, | |
| "codex_usage": { | |
| "accounting": "reported-responses", | |
| "audit_complete": true, | |
| "cache_hit_rate": 0.9695526652578946, | |
| "cache_reported_input_tokens": 1343960, | |
| "cache_write_input_tokens": 0, | |
| "cache_write_reported_input_tokens": 1343960, | |
| "cached_input_tokens": 1303040, | |
| "completed_turns": 1, | |
| "cost_usd": null, | |
| "failed_turns": 0, | |
| "input_tokens": 1343960, | |
| "known_cache_write_input_tokens": 0, | |
| "known_cached_input_tokens": 1303040, | |
| "known_input_tokens": 1343960, | |
| "known_output_tokens": 10223, | |
| "known_reasoning_output_tokens": 3099, | |
| "output_tokens": 10223, | |
| "reasoning_output_tokens": 3099, | |
| "reasoning_reported_output_tokens": 10223, | |
| "reported_responses": { | |
| "cache_reported_input_tokens": 39, | |
| "cache_write_input_tokens": 39, | |
| "cache_write_reported_input_tokens": 39, | |
| "cached_input_tokens": 39, | |
| "input_tokens": 39, | |
| "output_tokens": 39, | |
| "reasoning_output_tokens": 39, | |
| "reasoning_reported_output_tokens": 39 | |
| }, | |
| "response_count": 39, | |
| "response_ids_complete": true, | |
| "schema": "rlebench/token-usage/1", | |
| "source": "Codex token_usage_record per response", | |
| "uncached_input_tokens": 40920, | |
| "unidentified_usage_records": 0 | |
| }, | |
| "kinex_usage": { | |
| "input_tokens": 6184301, | |
| "output_tokens": 24218, | |
| "cached_input_tokens": 6022016, | |
| "cache_write_input_tokens": 0, | |
| "reasoning_output_tokens": 9092, | |
| "cache_reported_input_tokens": 6184301, | |
| "cache_write_reported_input_tokens": 6184301, | |
| "reasoning_reported_output_tokens": 24218, | |
| "schema": "rlebench/token-usage/1", | |
| "source": "Provider terminal response usage", | |
| "response_count": 132, | |
| "reported_responses": { | |
| "input_tokens": 132, | |
| "output_tokens": 132, | |
| "cached_input_tokens": 132, | |
| "cache_write_input_tokens": 132, | |
| "reasoning_output_tokens": 132, | |
| "cache_reported_input_tokens": 132, | |
| "cache_write_reported_input_tokens": 132, | |
| "reasoning_reported_output_tokens": 132 | |
| }, | |
| "known_cached_input_tokens": 6022016, | |
| "known_input_tokens": 6184301, | |
| "known_output_tokens": 24218, | |
| "known_cache_write_input_tokens": 0, | |
| "known_reasoning_output_tokens": 9092, | |
| "cache_hit_rate": 0.9737585541195359, | |
| "uncached_input_tokens": 162285, | |
| "cost_usd": null, | |
| "response_ids_complete": true, | |
| "usage_source": "provider-raw", | |
| "request_attempts": 133, | |
| "rejected_attempts": 0, | |
| "unresolved_attempts": [], | |
| "audit_complete": true, | |
| "accounting": "reported-responses" | |
| } | |
| }, | |
| { | |
| "task_key": "task03/08", | |
| "codex_success": true, | |
| "kinex_success": true, | |
| "codex_steps": 433, | |
| "kinex_steps": 1399, | |
| "same_instruction": true, | |
| "same_instruction_policy": true, | |
| "same_roboenv_source": false, | |
| "codex_usage": { | |
| "accounting": "reported-responses", | |
| "audit_complete": true, | |
| "cache_hit_rate": 0.9536282677980344, | |
| "cache_reported_input_tokens": 683067, | |
| "cache_write_input_tokens": 0, | |
| "cache_write_reported_input_tokens": 683067, | |
| "cached_input_tokens": 651392, | |
| "completed_turns": 1, | |
| "cost_usd": null, | |
| "failed_turns": 0, | |
| "input_tokens": 683067, | |
| "known_cache_write_input_tokens": 0, | |
| "known_cached_input_tokens": 651392, | |
| "known_input_tokens": 683067, | |
| "known_output_tokens": 5483, | |
| "known_reasoning_output_tokens": 994, | |
| "output_tokens": 5483, | |
| "reasoning_output_tokens": 994, | |
| "reasoning_reported_output_tokens": 5483, | |
| "reported_responses": { | |
| "cache_reported_input_tokens": 22, | |
| "cache_write_input_tokens": 22, | |
| "cache_write_reported_input_tokens": 22, | |
| "cached_input_tokens": 22, | |
| "input_tokens": 22, | |
| "output_tokens": 22, | |
| "reasoning_output_tokens": 22, | |
| "reasoning_reported_output_tokens": 22 | |
| }, | |
| "response_count": 22, | |
| "response_ids_complete": true, | |
| "schema": "rlebench/token-usage/1", | |
| "source": "Codex token_usage_record per response", | |
| "uncached_input_tokens": 31675, | |
| "unidentified_usage_records": 0 | |
| }, | |
| "kinex_usage": { | |
| "input_tokens": 1051290, | |
| "output_tokens": 6748, | |
| "cached_input_tokens": 972160, | |
| "cache_write_input_tokens": 0, | |
| "reasoning_output_tokens": 1687, | |
| "cache_reported_input_tokens": 1051290, | |
| "cache_write_reported_input_tokens": 1051290, | |
| "reasoning_reported_output_tokens": 6748, | |
| "schema": "rlebench/token-usage/1", | |
| "source": "Provider terminal response usage", | |
| "response_count": 36, | |
| "reported_responses": { | |
| "input_tokens": 36, | |
| "output_tokens": 36, | |
| "cached_input_tokens": 36, | |
| "cache_write_input_tokens": 36, | |
| "reasoning_output_tokens": 36, | |
| "cache_reported_input_tokens": 36, | |
| "cache_write_reported_input_tokens": 36, | |
| "reasoning_reported_output_tokens": 36 | |
| }, | |
| "known_cached_input_tokens": 972160, | |
| "known_input_tokens": 1051290, | |
| "known_output_tokens": 6748, | |
| "known_cache_write_input_tokens": 0, | |
| "known_reasoning_output_tokens": 1687, | |
| "cache_hit_rate": 0.9247305691103311, | |
| "uncached_input_tokens": 79130, | |
| "cost_usd": null, | |
| "response_ids_complete": true, | |
| "usage_source": "provider-raw", | |
| "request_attempts": 36, | |
| "rejected_attempts": 0, | |
| "unresolved_attempts": [], | |
| "audit_complete": true | |
| } | |
| }, | |
| { | |
| "task_key": "task03/09", | |
| "codex_success": true, | |
| "kinex_success": true, | |
| "codex_steps": 1097, | |
| "kinex_steps": 2705, | |
| "same_instruction": false, | |
| "same_instruction_policy": false, | |
| "same_roboenv_source": false, | |
| "codex_usage": { | |
| "accounting": "reported-responses", | |
| "audit_complete": true, | |
| "cache_hit_rate": 0.9593260437553782, | |
| "cache_reported_input_tokens": 1228329, | |
| "cache_write_input_tokens": 0, | |
| "cache_write_reported_input_tokens": 1228329, | |
| "cached_input_tokens": 1178368, | |
| "completed_turns": 1, | |
| "cost_usd": null, | |
| "failed_turns": 0, | |
| "input_tokens": 1228329, | |
| "known_cache_write_input_tokens": 0, | |
| "known_cached_input_tokens": 1178368, | |
| "known_input_tokens": 1228329, | |
| "known_output_tokens": 8401, | |
| "known_reasoning_output_tokens": 2418, | |
| "output_tokens": 8401, | |
| "reasoning_output_tokens": 2418, | |
| "reasoning_reported_output_tokens": 8401, | |
| "reported_responses": { | |
| "cache_reported_input_tokens": 34, | |
| "cache_write_input_tokens": 34, | |
| "cache_write_reported_input_tokens": 34, | |
| "cached_input_tokens": 34, | |
| "input_tokens": 34, | |
| "output_tokens": 34, | |
| "reasoning_output_tokens": 34, | |
| "reasoning_reported_output_tokens": 34 | |
| }, | |
| "response_count": 34, | |
| "response_ids_complete": true, | |
| "schema": "rlebench/token-usage/1", | |
| "source": "Codex token_usage_record per response", | |
| "uncached_input_tokens": 49961, | |
| "unidentified_usage_records": 0 | |
| }, | |
| "kinex_usage": { | |
| "input_tokens": 1739536, | |
| "output_tokens": 8301, | |
| "cached_input_tokens": 1646592, | |
| "cache_write_input_tokens": 0, | |
| "reasoning_output_tokens": 2232, | |
| "cache_reported_input_tokens": 1739536, | |
| "cache_write_reported_input_tokens": 1739536, | |
| "reasoning_reported_output_tokens": 8301, | |
| "schema": "rlebench/token-usage/1", | |
| "source": "Provider terminal response usage", | |
| "response_count": 53, | |
| "reported_responses": { | |
| "input_tokens": 53, | |
| "output_tokens": 53, | |
| "cached_input_tokens": 53, | |
| "cache_write_input_tokens": 53, | |
| "reasoning_output_tokens": 53, | |
| "cache_reported_input_tokens": 53, | |
| "cache_write_reported_input_tokens": 53, | |
| "reasoning_reported_output_tokens": 53 | |
| }, | |
| "known_cached_input_tokens": 1646592, | |
| "known_input_tokens": 1739536, | |
| "known_output_tokens": 8301, | |
| "known_cache_write_input_tokens": 0, | |
| "known_reasoning_output_tokens": 2232, | |
| "cache_hit_rate": 0.9465696599552984, | |
| "uncached_input_tokens": 92944, | |
| "cost_usd": null, | |
| "response_ids_complete": true, | |
| "usage_source": "provider-raw", | |
| "request_attempts": 53, | |
| "rejected_attempts": 0, | |
| "unresolved_attempts": [], | |
| "audit_complete": true | |
| } | |
| } | |
| ], | |
| "limits": [ | |
| "Single seed and one attempt per task. Historical Kinex references may have different source or instruction policy; equality is reported per task.", | |
| "GPU concurrency differs; wall time is not a controlled speed comparison.", | |
| "Lift pot is deferred and excluded from the new Codex results." | |
| ] | |
| }, | |
| "native_tcp_lift_pot": { | |
| "task_key": "task03/10", | |
| "same_roboenv_source": true, | |
| "same_instruction": true, | |
| "seed": 0, | |
| "kinex_success": true, | |
| "codex_success": true, | |
| "kinex_steps": 571, | |
| "codex_steps": 648, | |
| "kinex_usage": { | |
| "input_tokens": 1135571, | |
| "output_tokens": 7221, | |
| "cached_input_tokens": 1052672, | |
| "cache_write_input_tokens": 0, | |
| "reasoning_output_tokens": 1498, | |
| "cache_reported_input_tokens": 1135571, | |
| "cache_write_reported_input_tokens": 1135571, | |
| "reasoning_reported_output_tokens": 7221, | |
| "schema": "rlebench/token-usage/1", | |
| "source": "Provider terminal response usage", | |
| "response_count": 38, | |
| "reported_responses": { | |
| "input_tokens": 38, | |
| "output_tokens": 38, | |
| "cached_input_tokens": 38, | |
| "cache_write_input_tokens": 38, | |
| "reasoning_output_tokens": 38, | |
| "cache_reported_input_tokens": 38, | |
| "cache_write_reported_input_tokens": 38, | |
| "reasoning_reported_output_tokens": 38 | |
| }, | |
| "known_cached_input_tokens": 1052672, | |
| "known_input_tokens": 1135571, | |
| "known_output_tokens": 7221, | |
| "known_cache_write_input_tokens": 0, | |
| "known_reasoning_output_tokens": 1498, | |
| "cache_hit_rate": 0.9269979596167919, | |
| "uncached_input_tokens": 82899, | |
| "cost_usd": null, | |
| "response_ids_complete": true, | |
| "accounting": "reported-responses", | |
| "usage_source": "provider-raw", | |
| "request_attempts": 38, | |
| "rejected_attempts": 0, | |
| "unresolved_attempts": [], | |
| "audit_complete": true | |
| }, | |
| "codex_usage": { | |
| "accounting": "reported-responses", | |
| "audit_complete": true, | |
| "cache_hit_rate": 0.9595922025816601, | |
| "cache_reported_input_tokens": 984018, | |
| "cache_write_input_tokens": 0, | |
| "cache_write_reported_input_tokens": 984018, | |
| "cached_input_tokens": 944256, | |
| "completed_turns": 1, | |
| "cost_usd": null, | |
| "failed_turns": 0, | |
| "input_tokens": 984018, | |
| "known_cache_write_input_tokens": 0, | |
| "known_cached_input_tokens": 944256, | |
| "known_input_tokens": 984018, | |
| "known_output_tokens": 8009, | |
| "known_reasoning_output_tokens": 1896, | |
| "output_tokens": 8009, | |
| "reasoning_output_tokens": 1896, | |
| "reasoning_reported_output_tokens": 8009, | |
| "reported_responses": { | |
| "cache_reported_input_tokens": 29, | |
| "cache_write_input_tokens": 29, | |
| "cache_write_reported_input_tokens": 29, | |
| "cached_input_tokens": 29, | |
| "input_tokens": 29, | |
| "output_tokens": 29, | |
| "reasoning_output_tokens": 29, | |
| "reasoning_reported_output_tokens": 29 | |
| }, | |
| "response_count": 29, | |
| "response_ids_complete": true, | |
| "schema": "rlebench/token-usage/1", | |
| "source": "Codex token_usage_record per response", | |
| "uncached_input_tokens": 39762, | |
| "unidentified_usage_records": 0 | |
| }, | |
| "kinex_reference_revisions": { | |
| "dataset": "a94cdd46298ff8a6d01f872db172ad2159253e99", | |
| "space": "e19c1631fa35a61dc80c50df8712c28469e28028" | |
| }, | |
| "limitation": "One episode per harness; this is not a multi-seed success-rate estimate." | |
| }, | |
| "simple": { | |
| "tasks": [ | |
| { | |
| "task_key": "task06/01", | |
| "kinex_success": true, | |
| "codex_success": true, | |
| "same_instruction": true, | |
| "same_budget": true, | |
| "same_roboenv_source": true, | |
| "kinex_steps": 2001, | |
| "codex_steps": 1647, | |
| "kinex_usage": { | |
| "input_tokens": 3121253, | |
| "output_tokens": 9863, | |
| "cached_input_tokens": 2984704, | |
| "cache_write_input_tokens": 0, | |
| "reasoning_output_tokens": 2815, | |
| "cache_reported_input_tokens": 3121253, | |
| "cache_write_reported_input_tokens": 3121253, | |
| "reasoning_reported_output_tokens": 9863, | |
| "schema": "rlebench/token-usage/1", | |
| "source": "Provider terminal response usage", | |
| "response_count": 84, | |
| "reported_responses": { | |
| "input_tokens": 84, | |
| "output_tokens": 84, | |
| "cached_input_tokens": 84, | |
| "cache_write_input_tokens": 84, | |
| "reasoning_output_tokens": 84, | |
| "cache_reported_input_tokens": 84, | |
| "cache_write_reported_input_tokens": 84, | |
| "reasoning_reported_output_tokens": 84 | |
| }, | |
| "known_cached_input_tokens": 2984704, | |
| "known_input_tokens": 3121253, | |
| "known_output_tokens": 9863, | |
| "known_cache_write_input_tokens": 0, | |
| "known_reasoning_output_tokens": 2815, | |
| "cache_hit_rate": 0.9562518642352927, | |
| "uncached_input_tokens": 136549, | |
| "cost_usd": null, | |
| "response_ids_complete": true, | |
| "accounting": "reported-responses", | |
| "usage_source": "provider-raw", | |
| "request_attempts": 84, | |
| "rejected_attempts": 0, | |
| "unresolved_attempts": [], | |
| "audit_complete": true | |
| }, | |
| "codex_usage": { | |
| "accounting": "reported-responses", | |
| "audit_complete": true, | |
| "cache_hit_rate": 0.9760161493873662, | |
| "cache_reported_input_tokens": 2168751, | |
| "cache_write_input_tokens": 0, | |
| "cache_write_reported_input_tokens": 2168751, | |
| "cached_input_tokens": 2116736, | |
| "completed_turns": 1, | |
| "cost_usd": null, | |
| "failed_turns": 0, | |
| "input_tokens": 2168751, | |
| "known_cache_write_input_tokens": 0, | |
| "known_cached_input_tokens": 2116736, | |
| "known_input_tokens": 2168751, | |
| "known_output_tokens": 10358, | |
| "known_reasoning_output_tokens": 2642, | |
| "output_tokens": 10358, | |
| "reasoning_output_tokens": 2642, | |
| "reasoning_reported_output_tokens": 10358, | |
| "reported_responses": { | |
| "cache_reported_input_tokens": 51, | |
| "cache_write_input_tokens": 51, | |
| "cache_write_reported_input_tokens": 51, | |
| "cached_input_tokens": 51, | |
| "input_tokens": 51, | |
| "output_tokens": 51, | |
| "reasoning_output_tokens": 51, | |
| "reasoning_reported_output_tokens": 51 | |
| }, | |
| "response_count": 51, | |
| "response_ids_complete": true, | |
| "schema": "rlebench/token-usage/1", | |
| "source": "Codex token_usage_record per response", | |
| "uncached_input_tokens": 52015, | |
| "unidentified_usage_records": 0 | |
| } | |
| }, | |
| { | |
| "task_key": "task06/02", | |
| "kinex_success": false, | |
| "codex_success": false, | |
| "same_instruction": true, | |
| "same_budget": true, | |
| "same_roboenv_source": true, | |
| "kinex_steps": 11590, | |
| "codex_steps": 3875, | |
| "kinex_usage": { | |
| "input_tokens": 22203467, | |
| "output_tokens": 31183, | |
| "cached_input_tokens": 21887104, | |
| "cache_write_input_tokens": 0, | |
| "reasoning_output_tokens": 12299, | |
| "cache_reported_input_tokens": 22203467, | |
| "cache_write_reported_input_tokens": 22203467, | |
| "reasoning_reported_output_tokens": 31183, | |
| "schema": "rlebench/token-usage/1", | |
| "source": "Provider terminal response usage", | |
| "response_count": 295, | |
| "reported_responses": { | |
| "input_tokens": 295, | |
| "output_tokens": 295, | |
| "cached_input_tokens": 295, | |
| "cache_write_input_tokens": 295, | |
| "reasoning_output_tokens": 295, | |
| "cache_reported_input_tokens": 295, | |
| "cache_write_reported_input_tokens": 295, | |
| "reasoning_reported_output_tokens": 295 | |
| }, | |
| "known_cached_input_tokens": 21887104, | |
| "known_input_tokens": 22203467, | |
| "known_output_tokens": 31183, | |
| "known_cache_write_input_tokens": 0, | |
| "known_reasoning_output_tokens": 12299, | |
| "cache_hit_rate": 0.9857516395975457, | |
| "uncached_input_tokens": 316363, | |
| "cost_usd": null, | |
| "response_ids_complete": true, | |
| "accounting": "reported-responses", | |
| "usage_source": "provider-raw", | |
| "request_attempts": 298, | |
| "rejected_attempts": 0, | |
| "unresolved_attempts": [], | |
| "audit_complete": true | |
| }, | |
| "codex_usage": { | |
| "accounting": "reported-responses", | |
| "audit_complete": true, | |
| "cache_hit_rate": 0.9820486387779543, | |
| "cache_reported_input_tokens": 4989928, | |
| "cache_write_input_tokens": 0, | |
| "cache_write_reported_input_tokens": 4989928, | |
| "cached_input_tokens": 4900352, | |
| "completed_turns": 1, | |
| "cost_usd": null, | |
| "failed_turns": 0, | |
| "input_tokens": 4989928, | |
| "known_cache_write_input_tokens": 0, | |
| "known_cached_input_tokens": 4900352, | |
| "known_input_tokens": 4989928, | |
| "known_output_tokens": 18385, | |
| "known_reasoning_output_tokens": 6657, | |
| "output_tokens": 18385, | |
| "reasoning_output_tokens": 6657, | |
| "reasoning_reported_output_tokens": 18385, | |
| "reported_responses": { | |
| "cache_reported_input_tokens": 90, | |
| "cache_write_input_tokens": 90, | |
| "cache_write_reported_input_tokens": 90, | |
| "cached_input_tokens": 90, | |
| "input_tokens": 90, | |
| "output_tokens": 90, | |
| "reasoning_output_tokens": 90, | |
| "reasoning_reported_output_tokens": 90 | |
| }, | |
| "response_count": 90, | |
| "response_ids_complete": true, | |
| "schema": "rlebench/token-usage/1", | |
| "source": "Codex token_usage_record per response", | |
| "uncached_input_tokens": 89576, | |
| "unidentified_usage_records": 0 | |
| } | |
| }, | |
| { | |
| "task_key": "task06/03", | |
| "kinex_success": true, | |
| "codex_success": false, | |
| "same_instruction": true, | |
| "same_budget": false, | |
| "same_roboenv_source": false, | |
| "kinex_steps": 1162, | |
| "codex_steps": 13225, | |
| "kinex_usage": { | |
| "input_tokens": 1081991, | |
| "output_tokens": 5296, | |
| "cached_input_tokens": 999296, | |
| "cache_write_input_tokens": 0, | |
| "reasoning_output_tokens": 1447, | |
| "cache_reported_input_tokens": 1081991, | |
| "cache_write_reported_input_tokens": 1081991, | |
| "reasoning_reported_output_tokens": 5296, | |
| "schema": "rlebench/token-usage/1", | |
| "source": "Provider terminal response usage", | |
| "response_count": 36, | |
| "reported_responses": { | |
| "input_tokens": 36, | |
| "output_tokens": 36, | |
| "cached_input_tokens": 36, | |
| "cache_write_input_tokens": 36, | |
| "reasoning_output_tokens": 36, | |
| "cache_reported_input_tokens": 36, | |
| "cache_write_reported_input_tokens": 36, | |
| "reasoning_reported_output_tokens": 36 | |
| }, | |
| "known_cached_input_tokens": 999296, | |
| "known_input_tokens": 1081991, | |
| "known_output_tokens": 5296, | |
| "known_cache_write_input_tokens": 0, | |
| "known_reasoning_output_tokens": 1447, | |
| "cache_hit_rate": 0.9235714529972985, | |
| "uncached_input_tokens": 82695, | |
| "cost_usd": null, | |
| "response_ids_complete": true, | |
| "accounting": "reported-responses", | |
| "usage_source": "provider-raw", | |
| "request_attempts": 36, | |
| "rejected_attempts": 0, | |
| "unresolved_attempts": [], | |
| "audit_complete": true | |
| }, | |
| "codex_usage": { | |
| "accounting": "reported-responses", | |
| "audit_complete": true, | |
| "cache_hit_rate": 0.9906796551805874, | |
| "cache_reported_input_tokens": 19912568, | |
| "cache_write_input_tokens": 0, | |
| "cache_write_reported_input_tokens": 19912568, | |
| "cached_input_tokens": 19726976, | |
| "completed_turns": 1, | |
| "cost_usd": null, | |
| "failed_turns": 0, | |
| "input_tokens": 19912568, | |
| "known_cache_write_input_tokens": 0, | |
| "known_cached_input_tokens": 19726976, | |
| "known_input_tokens": 19912568, | |
| "known_output_tokens": 53777, | |
| "known_reasoning_output_tokens": 29418, | |
| "output_tokens": 53777, | |
| "reasoning_output_tokens": 29418, | |
| "reasoning_reported_output_tokens": 53777, | |
| "reported_responses": { | |
| "cache_reported_input_tokens": 199, | |
| "cache_write_input_tokens": 199, | |
| "cache_write_reported_input_tokens": 199, | |
| "cached_input_tokens": 199, | |
| "input_tokens": 199, | |
| "output_tokens": 199, | |
| "reasoning_output_tokens": 199, | |
| "reasoning_reported_output_tokens": 199 | |
| }, | |
| "response_count": 199, | |
| "response_ids_complete": true, | |
| "schema": "rlebench/token-usage/1", | |
| "source": "Codex token_usage_record per response", | |
| "uncached_input_tokens": 185592, | |
| "unidentified_usage_records": 0 | |
| } | |
| }, | |
| { | |
| "task_key": "task06/04", | |
| "kinex_success": true, | |
| "codex_success": true, | |
| "same_instruction": true, | |
| "same_budget": false, | |
| "same_roboenv_source": false, | |
| "kinex_steps": 5252, | |
| "codex_steps": 116, | |
| "kinex_usage": { | |
| "input_tokens": 6817957, | |
| "output_tokens": 18011, | |
| "cached_input_tokens": 6643968, | |
| "cache_write_input_tokens": 0, | |
| "reasoning_output_tokens": 8769, | |
| "cache_reported_input_tokens": 6817957, | |
| "cache_write_reported_input_tokens": 6817957, | |
| "reasoning_reported_output_tokens": 18011, | |
| "schema": "rlebench/token-usage/1", | |
| "source": "Provider terminal response usage", | |
| "response_count": 127, | |
| "reported_responses": { | |
| "input_tokens": 127, | |
| "output_tokens": 127, | |
| "cached_input_tokens": 127, | |
| "cache_write_input_tokens": 127, | |
| "reasoning_output_tokens": 127, | |
| "cache_reported_input_tokens": 127, | |
| "cache_write_reported_input_tokens": 127, | |
| "reasoning_reported_output_tokens": 127 | |
| }, | |
| "known_cached_input_tokens": 6643968, | |
| "known_input_tokens": 6817957, | |
| "known_output_tokens": 18011, | |
| "known_cache_write_input_tokens": 0, | |
| "known_reasoning_output_tokens": 8769, | |
| "cache_hit_rate": 0.9744807718793181, | |
| "uncached_input_tokens": 173989, | |
| "cost_usd": null, | |
| "response_ids_complete": true, | |
| "accounting": "reported-responses", | |
| "usage_source": "provider-raw", | |
| "request_attempts": 127, | |
| "rejected_attempts": 0, | |
| "unresolved_attempts": [], | |
| "audit_complete": true | |
| }, | |
| "codex_usage": { | |
| "accounting": "reported-responses", | |
| "audit_complete": true, | |
| "cache_hit_rate": 0.9163462297790655, | |
| "cache_reported_input_tokens": 329238, | |
| "cache_write_input_tokens": 0, | |
| "cache_write_reported_input_tokens": 329238, | |
| "cached_input_tokens": 301696, | |
| "completed_turns": 1, | |
| "cost_usd": null, | |
| "failed_turns": 0, | |
| "input_tokens": 329238, | |
| "known_cache_write_input_tokens": 0, | |
| "known_cached_input_tokens": 301696, | |
| "known_input_tokens": 329238, | |
| "known_output_tokens": 2595, | |
| "known_reasoning_output_tokens": 187, | |
| "output_tokens": 2595, | |
| "reasoning_output_tokens": 187, | |
| "reasoning_reported_output_tokens": 2595, | |
| "reported_responses": { | |
| "cache_reported_input_tokens": 11, | |
| "cache_write_input_tokens": 11, | |
| "cache_write_reported_input_tokens": 11, | |
| "cached_input_tokens": 11, | |
| "input_tokens": 11, | |
| "output_tokens": 11, | |
| "reasoning_output_tokens": 11, | |
| "reasoning_reported_output_tokens": 11 | |
| }, | |
| "response_count": 11, | |
| "response_ids_complete": true, | |
| "schema": "rlebench/token-usage/1", | |
| "source": "Codex token_usage_record per response", | |
| "uncached_input_tokens": 27542, | |
| "unidentified_usage_records": 0 | |
| } | |
| }, | |
| { | |
| "task_key": "task06/05", | |
| "kinex_success": true, | |
| "codex_success": true, | |
| "same_instruction": true, | |
| "same_budget": false, | |
| "same_roboenv_source": false, | |
| "kinex_steps": 1700, | |
| "codex_steps": 12320, | |
| "kinex_usage": { | |
| "input_tokens": 3345117, | |
| "output_tokens": 13215, | |
| "cached_input_tokens": 3214080, | |
| "cache_write_input_tokens": 0, | |
| "reasoning_output_tokens": 5259, | |
| "cache_reported_input_tokens": 3345117, | |
| "cache_write_reported_input_tokens": 3345117, | |
| "reasoning_reported_output_tokens": 13215, | |
| "schema": "rlebench/token-usage/1", | |
| "source": "Provider terminal response usage", | |
| "response_count": 84, | |
| "reported_responses": { | |
| "input_tokens": 84, | |
| "output_tokens": 84, | |
| "cached_input_tokens": 84, | |
| "cache_write_input_tokens": 84, | |
| "reasoning_output_tokens": 84, | |
| "cache_reported_input_tokens": 84, | |
| "cache_write_reported_input_tokens": 84, | |
| "reasoning_reported_output_tokens": 84 | |
| }, | |
| "known_cached_input_tokens": 3214080, | |
| "known_input_tokens": 3345117, | |
| "known_output_tokens": 13215, | |
| "known_cache_write_input_tokens": 0, | |
| "known_reasoning_output_tokens": 5259, | |
| "cache_hit_rate": 0.9608273791320303, | |
| "uncached_input_tokens": 131037, | |
| "cost_usd": null, | |
| "response_ids_complete": true, | |
| "accounting": "reported-responses", | |
| "usage_source": "provider-raw", | |
| "request_attempts": 84, | |
| "rejected_attempts": 0, | |
| "unresolved_attempts": [], | |
| "audit_complete": true | |
| }, | |
| "codex_usage": { | |
| "accounting": "reported-responses", | |
| "audit_complete": true, | |
| "cache_hit_rate": 0.9926789916948668, | |
| "cache_reported_input_tokens": 32371907, | |
| "cache_write_input_tokens": 0, | |
| "cache_write_reported_input_tokens": 32371907, | |
| "cached_input_tokens": 32134912, | |
| "completed_turns": 1, | |
| "cost_usd": null, | |
| "failed_turns": 0, | |
| "input_tokens": 32371907, | |
| "known_cache_write_input_tokens": 0, | |
| "known_cached_input_tokens": 32134912, | |
| "known_input_tokens": 32371907, | |
| "known_output_tokens": 103454, | |
| "known_reasoning_output_tokens": 69820, | |
| "output_tokens": 103454, | |
| "reasoning_output_tokens": 69820, | |
| "reasoning_reported_output_tokens": 103454, | |
| "reported_responses": { | |
| "cache_reported_input_tokens": 297, | |
| "cache_write_input_tokens": 297, | |
| "cache_write_reported_input_tokens": 297, | |
| "cached_input_tokens": 297, | |
| "input_tokens": 297, | |
| "output_tokens": 297, | |
| "reasoning_output_tokens": 297, | |
| "reasoning_reported_output_tokens": 297 | |
| }, | |
| "response_count": 297, | |
| "response_ids_complete": true, | |
| "schema": "rlebench/token-usage/1", | |
| "source": "Codex token_usage_record per response", | |
| "uncached_input_tokens": 236995, | |
| "unidentified_usage_records": 0 | |
| } | |
| }, | |
| { | |
| "task_key": "task06/06", | |
| "kinex_success": false, | |
| "codex_success": false, | |
| "same_instruction": true, | |
| "same_budget": false, | |
| "same_roboenv_source": false, | |
| "kinex_steps": 8480, | |
| "codex_steps": 8170, | |
| "kinex_usage": { | |
| "input_tokens": 21133569, | |
| "output_tokens": 34940, | |
| "cached_input_tokens": 20867712, | |
| "cache_write_input_tokens": 0, | |
| "reasoning_output_tokens": 15209, | |
| "cache_reported_input_tokens": 21133569, | |
| "cache_write_reported_input_tokens": 21133569, | |
| "reasoning_reported_output_tokens": 34940, | |
| "schema": "rlebench/token-usage/1", | |
| "source": "Provider terminal response usage", | |
| "response_count": 277, | |
| "reported_responses": { | |
| "input_tokens": 277, | |
| "output_tokens": 277, | |
| "cached_input_tokens": 277, | |
| "cache_write_input_tokens": 277, | |
| "reasoning_output_tokens": 277, | |
| "cache_reported_input_tokens": 277, | |
| "cache_write_reported_input_tokens": 277, | |
| "reasoning_reported_output_tokens": 277 | |
| }, | |
| "known_cached_input_tokens": 20867712, | |
| "known_input_tokens": 21133569, | |
| "known_output_tokens": 34940, | |
| "known_cache_write_input_tokens": 0, | |
| "known_reasoning_output_tokens": 15209, | |
| "cache_hit_rate": 0.9874201560559884, | |
| "uncached_input_tokens": 265857, | |
| "cost_usd": null, | |
| "response_ids_complete": true, | |
| "accounting": "reported-responses", | |
| "usage_source": "provider-raw", | |
| "request_attempts": 278, | |
| "rejected_attempts": 0, | |
| "unresolved_attempts": [], | |
| "audit_complete": true | |
| }, | |
| "codex_usage": { | |
| "accounting": "reported-responses", | |
| "audit_complete": true, | |
| "cache_hit_rate": 0.9716815288356057, | |
| "cache_reported_input_tokens": 8606079, | |
| "cache_write_input_tokens": 0, | |
| "cache_write_reported_input_tokens": 8606079, | |
| "cached_input_tokens": 8362368, | |
| "completed_turns": 1, | |
| "cost_usd": null, | |
| "failed_turns": 0, | |
| "input_tokens": 8606079, | |
| "known_cache_write_input_tokens": 0, | |
| "known_cached_input_tokens": 8362368, | |
| "known_input_tokens": 8606079, | |
| "known_output_tokens": 24634, | |
| "known_reasoning_output_tokens": 9316, | |
| "output_tokens": 24634, | |
| "reasoning_output_tokens": 9316, | |
| "reasoning_reported_output_tokens": 24634, | |
| "reported_responses": { | |
| "cache_reported_input_tokens": 134, | |
| "cache_write_input_tokens": 134, | |
| "cache_write_reported_input_tokens": 134, | |
| "cached_input_tokens": 134, | |
| "input_tokens": 134, | |
| "output_tokens": 134, | |
| "reasoning_output_tokens": 134, | |
| "reasoning_reported_output_tokens": 134 | |
| }, | |
| "response_count": 134, | |
| "response_ids_complete": true, | |
| "schema": "rlebench/token-usage/1", | |
| "source": "Codex token_usage_record per response", | |
| "uncached_input_tokens": 243711, | |
| "unidentified_usage_records": 0 | |
| } | |
| }, | |
| { | |
| "task_key": "task06/07", | |
| "kinex_success": false, | |
| "codex_success": false, | |
| "same_instruction": true, | |
| "same_budget": true, | |
| "same_roboenv_source": true, | |
| "kinex_steps": 12022, | |
| "codex_steps": 13688, | |
| "kinex_usage": { | |
| "input_tokens": 27597531, | |
| "output_tokens": 39270, | |
| "cached_input_tokens": 27060864, | |
| "cache_write_input_tokens": 0, | |
| "reasoning_output_tokens": 15762, | |
| "cache_reported_input_tokens": 27597531, | |
| "cache_write_reported_input_tokens": 27597531, | |
| "reasoning_reported_output_tokens": 39270, | |
| "schema": "rlebench/token-usage/1", | |
| "source": "Provider terminal response usage", | |
| "response_count": 315, | |
| "reported_responses": { | |
| "input_tokens": 315, | |
| "output_tokens": 315, | |
| "cached_input_tokens": 315, | |
| "cache_write_input_tokens": 315, | |
| "reasoning_output_tokens": 315, | |
| "cache_reported_input_tokens": 315, | |
| "cache_write_reported_input_tokens": 315, | |
| "reasoning_reported_output_tokens": 315 | |
| }, | |
| "known_cached_input_tokens": 27060864, | |
| "known_input_tokens": 27597531, | |
| "known_output_tokens": 39270, | |
| "known_cache_write_input_tokens": 0, | |
| "known_reasoning_output_tokens": 15762, | |
| "cache_hit_rate": 0.9805538038891958, | |
| "uncached_input_tokens": 536667, | |
| "cost_usd": null, | |
| "response_ids_complete": true, | |
| "accounting": "reported-responses", | |
| "usage_source": "provider-raw", | |
| "request_attempts": 318, | |
| "rejected_attempts": 0, | |
| "unresolved_attempts": [], | |
| "audit_complete": true | |
| }, | |
| "codex_usage": { | |
| "accounting": "reported-responses", | |
| "audit_complete": true, | |
| "cache_hit_rate": 0.9903104416710947, | |
| "cache_reported_input_tokens": 28195506, | |
| "cache_write_input_tokens": 0, | |
| "cache_write_reported_input_tokens": 28195506, | |
| "cached_input_tokens": 27922304, | |
| "completed_turns": 1, | |
| "cost_usd": null, | |
| "failed_turns": 0, | |
| "input_tokens": 28195506, | |
| "known_cache_write_input_tokens": 0, | |
| "known_cached_input_tokens": 27922304, | |
| "known_input_tokens": 28195506, | |
| "known_output_tokens": 45427, | |
| "known_reasoning_output_tokens": 17604, | |
| "output_tokens": 45427, | |
| "reasoning_output_tokens": 17604, | |
| "reasoning_reported_output_tokens": 45427, | |
| "reported_responses": { | |
| "cache_reported_input_tokens": 330, | |
| "cache_write_input_tokens": 330, | |
| "cache_write_reported_input_tokens": 330, | |
| "cached_input_tokens": 330, | |
| "input_tokens": 330, | |
| "output_tokens": 330, | |
| "reasoning_output_tokens": 330, | |
| "reasoning_reported_output_tokens": 330 | |
| }, | |
| "response_count": 330, | |
| "response_ids_complete": true, | |
| "schema": "rlebench/token-usage/1", | |
| "source": "Codex token_usage_record per response", | |
| "uncached_input_tokens": 273202, | |
| "unidentified_usage_records": 0 | |
| } | |
| }, | |
| { | |
| "task_key": "task06/08", | |
| "kinex_success": true, | |
| "codex_success": true, | |
| "same_instruction": true, | |
| "same_budget": false, | |
| "same_roboenv_source": false, | |
| "kinex_steps": 5077, | |
| "codex_steps": 1277, | |
| "kinex_usage": { | |
| "input_tokens": 5518859, | |
| "output_tokens": 11779, | |
| "cached_input_tokens": 5384064, | |
| "cache_write_input_tokens": 0, | |
| "reasoning_output_tokens": 4546, | |
| "cache_reported_input_tokens": 5518859, | |
| "cache_write_reported_input_tokens": 5518859, | |
| "reasoning_reported_output_tokens": 11779, | |
| "schema": "rlebench/token-usage/1", | |
| "source": "Provider terminal response usage", | |
| "response_count": 143, | |
| "reported_responses": { | |
| "input_tokens": 143, | |
| "output_tokens": 143, | |
| "cached_input_tokens": 143, | |
| "cache_write_input_tokens": 143, | |
| "reasoning_output_tokens": 143, | |
| "cache_reported_input_tokens": 143, | |
| "cache_write_reported_input_tokens": 143, | |
| "reasoning_reported_output_tokens": 143 | |
| }, | |
| "known_cached_input_tokens": 5384064, | |
| "known_input_tokens": 5518859, | |
| "known_output_tokens": 11779, | |
| "known_cache_write_input_tokens": 0, | |
| "known_reasoning_output_tokens": 4546, | |
| "cache_hit_rate": 0.9755755673410029, | |
| "uncached_input_tokens": 134795, | |
| "cost_usd": null, | |
| "response_ids_complete": true, | |
| "accounting": "reported-responses", | |
| "usage_source": "provider-raw", | |
| "request_attempts": 143, | |
| "rejected_attempts": 0, | |
| "unresolved_attempts": [], | |
| "audit_complete": true | |
| }, | |
| "codex_usage": { | |
| "accounting": "reported-responses", | |
| "audit_complete": true, | |
| "cache_hit_rate": 0.9652520364738014, | |
| "cache_reported_input_tokens": 1340798, | |
| "cache_write_input_tokens": 0, | |
| "cache_write_reported_input_tokens": 1340798, | |
| "cached_input_tokens": 1294208, | |
| "completed_turns": 1, | |
| "cost_usd": null, | |
| "failed_turns": 0, | |
| "input_tokens": 1340798, | |
| "known_cache_write_input_tokens": 0, | |
| "known_cached_input_tokens": 1294208, | |
| "known_input_tokens": 1340798, | |
| "known_output_tokens": 6729, | |
| "known_reasoning_output_tokens": 2707, | |
| "output_tokens": 6729, | |
| "reasoning_output_tokens": 2707, | |
| "reasoning_reported_output_tokens": 6729, | |
| "reported_responses": { | |
| "cache_reported_input_tokens": 32, | |
| "cache_write_input_tokens": 32, | |
| "cache_write_reported_input_tokens": 32, | |
| "cached_input_tokens": 32, | |
| "input_tokens": 32, | |
| "output_tokens": 32, | |
| "reasoning_output_tokens": 32, | |
| "reasoning_reported_output_tokens": 32 | |
| }, | |
| "response_count": 32, | |
| "response_ids_complete": true, | |
| "schema": "rlebench/token-usage/1", | |
| "source": "Codex token_usage_record per response", | |
| "uncached_input_tokens": 46590, | |
| "unidentified_usage_records": 0 | |
| } | |
| }, | |
| { | |
| "task_key": "task06/09", | |
| "kinex_success": true, | |
| "codex_success": false, | |
| "same_instruction": true, | |
| "same_budget": true, | |
| "same_roboenv_source": true, | |
| "kinex_steps": 2155, | |
| "codex_steps": 3170, | |
| "kinex_usage": { | |
| "input_tokens": 1946704, | |
| "output_tokens": 6386, | |
| "cached_input_tokens": 1787648, | |
| "cache_write_input_tokens": 0, | |
| "reasoning_output_tokens": 1669, | |
| "cache_reported_input_tokens": 1946704, | |
| "cache_write_reported_input_tokens": 1946704, | |
| "reasoning_reported_output_tokens": 6386, | |
| "schema": "rlebench/token-usage/1", | |
| "source": "Provider terminal response usage", | |
| "response_count": 64, | |
| "reported_responses": { | |
| "input_tokens": 64, | |
| "output_tokens": 64, | |
| "cached_input_tokens": 64, | |
| "cache_write_input_tokens": 64, | |
| "reasoning_output_tokens": 64, | |
| "cache_reported_input_tokens": 64, | |
| "cache_write_reported_input_tokens": 64, | |
| "reasoning_reported_output_tokens": 64 | |
| }, | |
| "known_cached_input_tokens": 1787648, | |
| "known_input_tokens": 1946704, | |
| "known_output_tokens": 6386, | |
| "known_cache_write_input_tokens": 0, | |
| "known_reasoning_output_tokens": 1669, | |
| "cache_hit_rate": 0.9182947176355521, | |
| "uncached_input_tokens": 159056, | |
| "cost_usd": null, | |
| "response_ids_complete": true, | |
| "accounting": "reported-responses", | |
| "usage_source": "provider-raw", | |
| "request_attempts": 64, | |
| "rejected_attempts": 0, | |
| "unresolved_attempts": [], | |
| "audit_complete": true | |
| }, | |
| "codex_usage": { | |
| "accounting": "reported-responses", | |
| "audit_complete": true, | |
| "cache_hit_rate": 0.9660542143458919, | |
| "cache_reported_input_tokens": 4133002, | |
| "cache_write_input_tokens": 0, | |
| "cache_write_reported_input_tokens": 4133002, | |
| "cached_input_tokens": 3992704, | |
| "completed_turns": 1, | |
| "cost_usd": null, | |
| "failed_turns": 0, | |
| "input_tokens": 4133002, | |
| "known_cache_write_input_tokens": 0, | |
| "known_cached_input_tokens": 3992704, | |
| "known_input_tokens": 4133002, | |
| "known_output_tokens": 16710, | |
| "known_reasoning_output_tokens": 5193, | |
| "output_tokens": 16710, | |
| "reasoning_output_tokens": 5193, | |
| "reasoning_reported_output_tokens": 16710, | |
| "reported_responses": { | |
| "cache_reported_input_tokens": 84, | |
| "cache_write_input_tokens": 84, | |
| "cache_write_reported_input_tokens": 84, | |
| "cached_input_tokens": 84, | |
| "input_tokens": 84, | |
| "output_tokens": 84, | |
| "reasoning_output_tokens": 84, | |
| "reasoning_reported_output_tokens": 84 | |
| }, | |
| "response_count": 84, | |
| "response_ids_complete": true, | |
| "schema": "rlebench/token-usage/1", | |
| "source": "Codex token_usage_record per response", | |
| "uncached_input_tokens": 140298, | |
| "unidentified_usage_records": 0 | |
| } | |
| }, | |
| { | |
| "task_key": "task06/10", | |
| "kinex_success": true, | |
| "codex_success": true, | |
| "same_instruction": true, | |
| "same_budget": true, | |
| "same_roboenv_source": true, | |
| "kinex_steps": 6641, | |
| "codex_steps": 1710, | |
| "kinex_usage": { | |
| "input_tokens": 5852385, | |
| "output_tokens": 12793, | |
| "cached_input_tokens": 5652608, | |
| "cache_write_input_tokens": 0, | |
| "reasoning_output_tokens": 4543, | |
| "cache_reported_input_tokens": 5852385, | |
| "cache_write_reported_input_tokens": 5852385, | |
| "reasoning_reported_output_tokens": 12793, | |
| "schema": "rlebench/token-usage/1", | |
| "source": "Provider terminal response usage", | |
| "response_count": 130, | |
| "reported_responses": { | |
| "input_tokens": 130, | |
| "output_tokens": 130, | |
| "cached_input_tokens": 130, | |
| "cache_write_input_tokens": 130, | |
| "reasoning_output_tokens": 130, | |
| "cache_reported_input_tokens": 130, | |
| "cache_write_reported_input_tokens": 130, | |
| "reasoning_reported_output_tokens": 130 | |
| }, | |
| "known_cached_input_tokens": 5652608, | |
| "known_input_tokens": 5852385, | |
| "known_output_tokens": 12793, | |
| "known_cache_write_input_tokens": 0, | |
| "known_reasoning_output_tokens": 4543, | |
| "cache_hit_rate": 0.9658640024537005, | |
| "uncached_input_tokens": 199777, | |
| "cost_usd": null, | |
| "response_ids_complete": true, | |
| "accounting": "reported-responses", | |
| "usage_source": "provider-raw", | |
| "request_attempts": 130, | |
| "rejected_attempts": 0, | |
| "unresolved_attempts": [], | |
| "audit_complete": true | |
| }, | |
| "codex_usage": { | |
| "accounting": "reported-responses", | |
| "audit_complete": true, | |
| "cache_hit_rate": 0.9782577578812951, | |
| "cache_reported_input_tokens": 2661777, | |
| "cache_write_input_tokens": 0, | |
| "cache_write_reported_input_tokens": 2661777, | |
| "cached_input_tokens": 2603904, | |
| "completed_turns": 1, | |
| "cost_usd": null, | |
| "failed_turns": 0, | |
| "input_tokens": 2661777, | |
| "known_cache_write_input_tokens": 0, | |
| "known_cached_input_tokens": 2603904, | |
| "known_input_tokens": 2661777, | |
| "known_output_tokens": 11612, | |
| "known_reasoning_output_tokens": 4313, | |
| "output_tokens": 11612, | |
| "reasoning_output_tokens": 4313, | |
| "reasoning_reported_output_tokens": 11612, | |
| "reported_responses": { | |
| "cache_reported_input_tokens": 62, | |
| "cache_write_input_tokens": 62, | |
| "cache_write_reported_input_tokens": 62, | |
| "cached_input_tokens": 62, | |
| "input_tokens": 62, | |
| "output_tokens": 62, | |
| "reasoning_output_tokens": 62, | |
| "reasoning_reported_output_tokens": 62 | |
| }, | |
| "response_count": 62, | |
| "response_ids_complete": true, | |
| "schema": "rlebench/token-usage/1", | |
| "source": "Codex token_usage_record per response", | |
| "uncached_input_tokens": 57873, | |
| "unidentified_usage_records": 0 | |
| } | |
| }, | |
| { | |
| "task_key": "task06/11", | |
| "kinex_success": true, | |
| "codex_success": true, | |
| "same_instruction": true, | |
| "same_budget": true, | |
| "same_roboenv_source": true, | |
| "kinex_steps": 360, | |
| "codex_steps": 581, | |
| "kinex_usage": { | |
| "input_tokens": 333016, | |
| "output_tokens": 1394, | |
| "cached_input_tokens": 271872, | |
| "cache_write_input_tokens": 0, | |
| "reasoning_output_tokens": 93, | |
| "cache_reported_input_tokens": 333016, | |
| "cache_write_reported_input_tokens": 333016, | |
| "reasoning_reported_output_tokens": 1394, | |
| "schema": "rlebench/token-usage/1", | |
| "source": "Provider terminal response usage", | |
| "response_count": 19, | |
| "reported_responses": { | |
| "input_tokens": 19, | |
| "output_tokens": 19, | |
| "cached_input_tokens": 19, | |
| "cache_write_input_tokens": 19, | |
| "reasoning_output_tokens": 19, | |
| "cache_reported_input_tokens": 19, | |
| "cache_write_reported_input_tokens": 19, | |
| "reasoning_reported_output_tokens": 19 | |
| }, | |
| "known_cached_input_tokens": 271872, | |
| "known_input_tokens": 333016, | |
| "known_output_tokens": 1394, | |
| "known_cache_write_input_tokens": 0, | |
| "known_reasoning_output_tokens": 93, | |
| "cache_hit_rate": 0.8163932063324285, | |
| "uncached_input_tokens": 61144, | |
| "cost_usd": null, | |
| "response_ids_complete": true, | |
| "accounting": "reported-responses", | |
| "usage_source": "provider-raw", | |
| "request_attempts": 19, | |
| "rejected_attempts": 0, | |
| "unresolved_attempts": [], | |
| "audit_complete": true | |
| }, | |
| "codex_usage": { | |
| "accounting": "reported-responses", | |
| "audit_complete": true, | |
| "cache_hit_rate": 0.9631537448295527, | |
| "cache_reported_input_tokens": 866221, | |
| "cache_write_input_tokens": 0, | |
| "cache_write_reported_input_tokens": 866221, | |
| "cached_input_tokens": 834304, | |
| "completed_turns": 1, | |
| "cost_usd": null, | |
| "failed_turns": 0, | |
| "input_tokens": 866221, | |
| "known_cache_write_input_tokens": 0, | |
| "known_cached_input_tokens": 834304, | |
| "known_input_tokens": 866221, | |
| "known_output_tokens": 2765, | |
| "known_reasoning_output_tokens": 543, | |
| "output_tokens": 2765, | |
| "reasoning_output_tokens": 543, | |
| "reasoning_reported_output_tokens": 2765, | |
| "reported_responses": { | |
| "cache_reported_input_tokens": 28, | |
| "cache_write_input_tokens": 28, | |
| "cache_write_reported_input_tokens": 28, | |
| "cached_input_tokens": 28, | |
| "input_tokens": 28, | |
| "output_tokens": 28, | |
| "reasoning_output_tokens": 28, | |
| "reasoning_reported_output_tokens": 28 | |
| }, | |
| "response_count": 28, | |
| "response_ids_complete": true, | |
| "schema": "rlebench/token-usage/1", | |
| "source": "Codex token_usage_record per response", | |
| "uncached_input_tokens": 31917, | |
| "unidentified_usage_records": 0 | |
| } | |
| }, | |
| { | |
| "task_key": "task06/12", | |
| "kinex_success": true, | |
| "codex_success": true, | |
| "same_instruction": true, | |
| "same_budget": true, | |
| "same_roboenv_source": true, | |
| "kinex_steps": 1114, | |
| "codex_steps": 1237, | |
| "kinex_usage": { | |
| "input_tokens": 1195670, | |
| "output_tokens": 4655, | |
| "cached_input_tokens": 1117952, | |
| "cache_write_input_tokens": 0, | |
| "reasoning_output_tokens": 1032, | |
| "cache_reported_input_tokens": 1195670, | |
| "cache_write_reported_input_tokens": 1195670, | |
| "reasoning_reported_output_tokens": 4655, | |
| "schema": "rlebench/token-usage/1", | |
| "source": "Provider terminal response usage", | |
| "response_count": 42, | |
| "reported_responses": { | |
| "input_tokens": 42, | |
| "output_tokens": 42, | |
| "cached_input_tokens": 42, | |
| "cache_write_input_tokens": 42, | |
| "reasoning_output_tokens": 42, | |
| "cache_reported_input_tokens": 42, | |
| "cache_write_reported_input_tokens": 42, | |
| "reasoning_reported_output_tokens": 42 | |
| }, | |
| "known_cached_input_tokens": 1117952, | |
| "known_input_tokens": 1195670, | |
| "known_output_tokens": 4655, | |
| "known_cache_write_input_tokens": 0, | |
| "known_reasoning_output_tokens": 1032, | |
| "cache_hit_rate": 0.935000459993142, | |
| "uncached_input_tokens": 77718, | |
| "cost_usd": null, | |
| "response_ids_complete": true, | |
| "accounting": "reported-responses", | |
| "usage_source": "provider-raw", | |
| "request_attempts": 42, | |
| "rejected_attempts": 0, | |
| "unresolved_attempts": [], | |
| "audit_complete": true | |
| }, | |
| "codex_usage": { | |
| "accounting": "reported-responses", | |
| "audit_complete": true, | |
| "cache_hit_rate": 0.9608732762844007, | |
| "cache_reported_input_tokens": 1187986, | |
| "cache_write_input_tokens": 0, | |
| "cache_write_reported_input_tokens": 1187986, | |
| "cached_input_tokens": 1141504, | |
| "completed_turns": 1, | |
| "cost_usd": null, | |
| "failed_turns": 0, | |
| "input_tokens": 1187986, | |
| "known_cache_write_input_tokens": 0, | |
| "known_cached_input_tokens": 1141504, | |
| "known_input_tokens": 1187986, | |
| "known_output_tokens": 7120, | |
| "known_reasoning_output_tokens": 2278, | |
| "output_tokens": 7120, | |
| "reasoning_output_tokens": 2278, | |
| "reasoning_reported_output_tokens": 7120, | |
| "reported_responses": { | |
| "cache_reported_input_tokens": 30, | |
| "cache_write_input_tokens": 30, | |
| "cache_write_reported_input_tokens": 30, | |
| "cached_input_tokens": 30, | |
| "input_tokens": 30, | |
| "output_tokens": 30, | |
| "reasoning_output_tokens": 30, | |
| "reasoning_reported_output_tokens": 30 | |
| }, | |
| "response_count": 30, | |
| "response_ids_complete": true, | |
| "schema": "rlebench/token-usage/1", | |
| "source": "Codex token_usage_record per response", | |
| "uncached_input_tokens": 46482, | |
| "unidentified_usage_records": 0 | |
| } | |
| } | |
| ], | |
| "limits": [ | |
| "One selected seed-0 result per agent and task. Earlier failures were explicitly rerun at 15000 steps; retained successes use 10000 steps. Old attempts remain in history. This is not a uniform-budget or controlled wall-time comparison.", | |
| "Pending or interrupted attempts are not imputed as successes or failures." | |
| ] | |
| } | |
| } | |