| { |
| "decisions": [ |
| { |
| "harness_id": "H000", |
| "role": "mandatory_exact_baseline" |
| }, |
| { |
| "harness_id": "H007", |
| "role": "highest_resolution" |
| }, |
| { |
| "harness_id": "H002", |
| "role": "lowest_tokens_within_5pp_resolution" |
| }, |
| { |
| "harness_id": "H014", |
| "role": "highest_accepted_edit" |
| }, |
| { |
| "harness_id": "H008", |
| "role": "best_prespecified_robustness_candidate" |
| }, |
| { |
| "harness_id": "H001", |
| "role": "pareto_frontier_fill" |
| } |
| ], |
| "e16_manifest_sha256": "ab444d5449b4b0cafce87c1afeb0036dd546496c824467ee40e7a2c79fbacb88", |
| "e16_planned_cells": 306, |
| "excluded_screening_experiment": { |
| "experiment_id": "E14", |
| "reason": "retrieval harnesses are repeated across action interfaces and are reserved for retrieval-by-action inference" |
| }, |
| "metrics": { |
| "H000": { |
| "accepted_edit_rate": 0.4666666666666667, |
| "applicable_patch_rate": 0.46111111111111114, |
| "cells": 180, |
| "mean_total_tokens": 72646.88333333333, |
| "mean_wall_seconds": 41.005881560401114, |
| "resolved_rate": 0.1 |
| }, |
| "H001": { |
| "accepted_edit_rate": 0.42777777777777776, |
| "applicable_patch_rate": 0.4111111111111111, |
| "cells": 180, |
| "mean_total_tokens": 72252.48333333334, |
| "mean_wall_seconds": 40.23372465622508, |
| "resolved_rate": 0.07777777777777778 |
| }, |
| "H002": { |
| "accepted_edit_rate": 0.3888888888888889, |
| "applicable_patch_rate": 0.37777777777777777, |
| "cells": 180, |
| "mean_total_tokens": 61987.822222222225, |
| "mean_wall_seconds": 40.5383593710411, |
| "resolved_rate": 0.08888888888888889 |
| }, |
| "H003": { |
| "accepted_edit_rate": 0.4222222222222222, |
| "applicable_patch_rate": 0.4166666666666667, |
| "cells": 180, |
| "mean_total_tokens": 74789.28888888888, |
| "mean_wall_seconds": 47.87801928565734, |
| "resolved_rate": 0.12222222222222222 |
| }, |
| "H004": { |
| "accepted_edit_rate": 0.46111111111111114, |
| "applicable_patch_rate": 0.45, |
| "cells": 180, |
| "mean_total_tokens": 68463.7, |
| "mean_wall_seconds": 47.970295593952244, |
| "resolved_rate": 0.1111111111111111 |
| }, |
| "H005": { |
| "accepted_edit_rate": 0.4111111111111111, |
| "applicable_patch_rate": 0.40555555555555556, |
| "cells": 180, |
| "mean_total_tokens": 74441.46666666666, |
| "mean_wall_seconds": 43.979250848839165, |
| "resolved_rate": 0.09444444444444444 |
| }, |
| "H006": { |
| "accepted_edit_rate": 0.45555555555555555, |
| "applicable_patch_rate": 0.4388888888888889, |
| "cells": 180, |
| "mean_total_tokens": 74994.5, |
| "mean_wall_seconds": 48.37345648450104, |
| "resolved_rate": 0.10555555555555556 |
| }, |
| "H007": { |
| "accepted_edit_rate": 0.43333333333333335, |
| "applicable_patch_rate": 0.42777777777777776, |
| "cells": 180, |
| "mean_total_tokens": 72679.36666666667, |
| "mean_wall_seconds": 47.19653170536427, |
| "resolved_rate": 0.12222222222222222 |
| }, |
| "H008": { |
| "accepted_edit_rate": 0.38333333333333336, |
| "applicable_patch_rate": 0.38333333333333336, |
| "cells": 60, |
| "mean_total_tokens": 66450.58333333333, |
| "mean_wall_seconds": 36.10224259162787, |
| "resolved_rate": 0.0 |
| }, |
| "H009": { |
| "accepted_edit_rate": 0.38333333333333336, |
| "applicable_patch_rate": 0.38333333333333336, |
| "cells": 60, |
| "mean_total_tokens": 63343.78333333333, |
| "mean_wall_seconds": 33.317234545220465, |
| "resolved_rate": 0.0 |
| }, |
| "H010": { |
| "accepted_edit_rate": 0.4166666666666667, |
| "applicable_patch_rate": 0.36666666666666664, |
| "cells": 60, |
| "mean_total_tokens": 87210.51666666666, |
| "mean_wall_seconds": 81.18024050413321, |
| "resolved_rate": 0.0 |
| }, |
| "H011": { |
| "accepted_edit_rate": 0.26666666666666666, |
| "applicable_patch_rate": 0.25, |
| "cells": 60, |
| "mean_total_tokens": 65144.8, |
| "mean_wall_seconds": 28.2178086270617, |
| "resolved_rate": 0.0 |
| }, |
| "H012": { |
| "accepted_edit_rate": 0.4166666666666667, |
| "applicable_patch_rate": 0.4, |
| "cells": 60, |
| "mean_total_tokens": 103753.21666666666, |
| "mean_wall_seconds": 48.84829152571425, |
| "resolved_rate": 0.0 |
| }, |
| "H013": { |
| "accepted_edit_rate": 0.36666666666666664, |
| "applicable_patch_rate": 0.36666666666666664, |
| "cells": 60, |
| "mean_total_tokens": 77243.36666666667, |
| "mean_wall_seconds": 38.010123237486305, |
| "resolved_rate": 0.0 |
| }, |
| "H014": { |
| "accepted_edit_rate": 0.4666666666666667, |
| "applicable_patch_rate": 0.45, |
| "cells": 60, |
| "mean_total_tokens": 145122.58333333334, |
| "mean_wall_seconds": 65.632639020142, |
| "resolved_rate": 0.0 |
| }, |
| "H015": { |
| "accepted_edit_rate": 0.35, |
| "applicable_patch_rate": 0.35, |
| "cells": 60, |
| "mean_total_tokens": 63847.38333333333, |
| "mean_wall_seconds": 35.46575253612439, |
| "resolved_rate": 0.0 |
| } |
| }, |
| "pareto_frontier": [ |
| "H000", |
| "H001", |
| "H002", |
| "H004", |
| "H007", |
| "H009", |
| "H011" |
| ], |
| "primary_screening_panel": { |
| "H000": "E13", |
| "H001": "E13", |
| "H002": "E13", |
| "H003": "E13", |
| "H004": "E13", |
| "H005": "E13", |
| "H006": "E13", |
| "H007": "E13", |
| "H008": "E15", |
| "H009": "E15", |
| "H010": "E15", |
| "H011": "E15", |
| "H012": "E15", |
| "H013": "E15", |
| "H014": "E15", |
| "H015": "E15" |
| }, |
| "robustness_candidates": [ |
| "H008", |
| "H010" |
| ], |
| "schema_version": 1, |
| "screening_experiments": [ |
| "E13", |
| "E15" |
| ], |
| "screening_revision": "7f4de67853deab34aca5a9ceaaf7e9f85b901088", |
| "selected_harnesses": [ |
| "H000", |
| "H001", |
| "H002", |
| "H007", |
| "H008", |
| "H014" |
| ], |
| "selection_is_outcome_aware_to_screening": true, |
| "selection_is_outcome_blind_to_e16": true, |
| "selection_sha256": "a6604699ba76e9eb95d31d5987363af0acd66d7ee0ed7d20e2eb64a7806b2e02", |
| "source_final_reports": { |
| "E13": { |
| "accepted_edit_count": 624, |
| "applicable_patch_count": 610, |
| "path": "results/reports/E13_7f4de67853de_1784618186.json", |
| "resolved_count": 148, |
| "run_count": 1440, |
| "sha256": "98cc58a125b5aa5c69e90b46134d9e01a93bd37d7d4f5d2160e0921746fb29a8" |
| }, |
| "E15": { |
| "accepted_edit_count": 206, |
| "applicable_patch_count": 199, |
| "path": "results/reports/E15_7f4de67853de_1784664649.json", |
| "resolved_count": 0, |
| "run_count": 540, |
| "sha256": "842cb293fd4269de8fafdb97b343d3baf78c789a54b94e8b3136d75216e83dab" |
| } |
| }, |
| "source_raw_sha256": { |
| "E13": "723a554fdb2596f0f8ece3d86f3f9b4333d2589bdae6e638b40798cb3b4c775e", |
| "E15": "30d40d4d371a8dac7554648f0066d85660d5486b3db8de686f7b66911d9f6e28" |
| }, |
| "study": "Study 5 end-to-end harness behavior" |
| } |
|
|
