{
  "decisions": [
    {
      "harness_id": "H000",
      "role": "mandatory_exact_baseline"
    },
    {
      "harness_id": "H007",
      "role": "highest_resolution"
    },
    {
      "harness_id": "H002",
      "role": "lowest_tokens_within_5pp_resolution"
    },
    {
      "harness_id": "H014",
      "role": "highest_accepted_edit"
    },
    {
      "harness_id": "H008",
      "role": "best_prespecified_robustness_candidate"
    },
    {
      "harness_id": "H001",
      "role": "pareto_frontier_fill"
    }
  ],
  "e16_manifest_sha256": "ab444d5449b4b0cafce87c1afeb0036dd546496c824467ee40e7a2c79fbacb88",
  "e16_planned_cells": 306,
  "excluded_screening_experiment": {
    "experiment_id": "E14",
    "reason": "retrieval harnesses are repeated across action interfaces and are reserved for retrieval-by-action inference"
  },
  "metrics": {
    "H000": {
      "accepted_edit_rate": 0.4666666666666667,
      "applicable_patch_rate": 0.46111111111111114,
      "cells": 180,
      "mean_total_tokens": 72646.88333333333,
      "mean_wall_seconds": 41.005881560401114,
      "resolved_rate": 0.1
    },
    "H001": {
      "accepted_edit_rate": 0.42777777777777776,
      "applicable_patch_rate": 0.4111111111111111,
      "cells": 180,
      "mean_total_tokens": 72252.48333333334,
      "mean_wall_seconds": 40.23372465622508,
      "resolved_rate": 0.07777777777777778
    },
    "H002": {
      "accepted_edit_rate": 0.3888888888888889,
      "applicable_patch_rate": 0.37777777777777777,
      "cells": 180,
      "mean_total_tokens": 61987.822222222225,
      "mean_wall_seconds": 40.5383593710411,
      "resolved_rate": 0.08888888888888889
    },
    "H003": {
      "accepted_edit_rate": 0.4222222222222222,
      "applicable_patch_rate": 0.4166666666666667,
      "cells": 180,
      "mean_total_tokens": 74789.28888888888,
      "mean_wall_seconds": 47.87801928565734,
      "resolved_rate": 0.12222222222222222
    },
    "H004": {
      "accepted_edit_rate": 0.46111111111111114,
      "applicable_patch_rate": 0.45,
      "cells": 180,
      "mean_total_tokens": 68463.7,
      "mean_wall_seconds": 47.970295593952244,
      "resolved_rate": 0.1111111111111111
    },
    "H005": {
      "accepted_edit_rate": 0.4111111111111111,
      "applicable_patch_rate": 0.40555555555555556,
      "cells": 180,
      "mean_total_tokens": 74441.46666666666,
      "mean_wall_seconds": 43.979250848839165,
      "resolved_rate": 0.09444444444444444
    },
    "H006": {
      "accepted_edit_rate": 0.45555555555555555,
      "applicable_patch_rate": 0.4388888888888889,
      "cells": 180,
      "mean_total_tokens": 74994.5,
      "mean_wall_seconds": 48.37345648450104,
      "resolved_rate": 0.10555555555555556
    },
    "H007": {
      "accepted_edit_rate": 0.43333333333333335,
      "applicable_patch_rate": 0.42777777777777776,
      "cells": 180,
      "mean_total_tokens": 72679.36666666667,
      "mean_wall_seconds": 47.19653170536427,
      "resolved_rate": 0.12222222222222222
    },
    "H008": {
      "accepted_edit_rate": 0.38333333333333336,
      "applicable_patch_rate": 0.38333333333333336,
      "cells": 60,
      "mean_total_tokens": 66450.58333333333,
      "mean_wall_seconds": 36.10224259162787,
      "resolved_rate": 0.0
    },
    "H009": {
      "accepted_edit_rate": 0.38333333333333336,
      "applicable_patch_rate": 0.38333333333333336,
      "cells": 60,
      "mean_total_tokens": 63343.78333333333,
      "mean_wall_seconds": 33.317234545220465,
      "resolved_rate": 0.0
    },
    "H010": {
      "accepted_edit_rate": 0.4166666666666667,
      "applicable_patch_rate": 0.36666666666666664,
      "cells": 60,
      "mean_total_tokens": 87210.51666666666,
      "mean_wall_seconds": 81.18024050413321,
      "resolved_rate": 0.0
    },
    "H011": {
      "accepted_edit_rate": 0.26666666666666666,
      "applicable_patch_rate": 0.25,
      "cells": 60,
      "mean_total_tokens": 65144.8,
      "mean_wall_seconds": 28.2178086270617,
      "resolved_rate": 0.0
    },
    "H012": {
      "accepted_edit_rate": 0.4166666666666667,
      "applicable_patch_rate": 0.4,
      "cells": 60,
      "mean_total_tokens": 103753.21666666666,
      "mean_wall_seconds": 48.84829152571425,
      "resolved_rate": 0.0
    },
    "H013": {
      "accepted_edit_rate": 0.36666666666666664,
      "applicable_patch_rate": 0.36666666666666664,
      "cells": 60,
      "mean_total_tokens": 77243.36666666667,
      "mean_wall_seconds": 38.010123237486305,
      "resolved_rate": 0.0
    },
    "H014": {
      "accepted_edit_rate": 0.4666666666666667,
      "applicable_patch_rate": 0.45,
      "cells": 60,
      "mean_total_tokens": 145122.58333333334,
      "mean_wall_seconds": 65.632639020142,
      "resolved_rate": 0.0
    },
    "H015": {
      "accepted_edit_rate": 0.35,
      "applicable_patch_rate": 0.35,
      "cells": 60,
      "mean_total_tokens": 63847.38333333333,
      "mean_wall_seconds": 35.46575253612439,
      "resolved_rate": 0.0
    }
  },
  "pareto_frontier": [
    "H000",
    "H001",
    "H002",
    "H004",
    "H007",
    "H009",
    "H011"
  ],
  "primary_screening_panel": {
    "H000": "E13",
    "H001": "E13",
    "H002": "E13",
    "H003": "E13",
    "H004": "E13",
    "H005": "E13",
    "H006": "E13",
    "H007": "E13",
    "H008": "E15",
    "H009": "E15",
    "H010": "E15",
    "H011": "E15",
    "H012": "E15",
    "H013": "E15",
    "H014": "E15",
    "H015": "E15"
  },
  "robustness_candidates": [
    "H008",
    "H010"
  ],
  "schema_version": 1,
  "screening_experiments": [
    "E13",
    "E15"
  ],
  "screening_revision": "7f4de67853deab34aca5a9ceaaf7e9f85b901088",
  "selected_harnesses": [
    "H000",
    "H001",
    "H002",
    "H007",
    "H008",
    "H014"
  ],
  "selection_is_outcome_aware_to_screening": true,
  "selection_is_outcome_blind_to_e16": true,
  "selection_sha256": "a6604699ba76e9eb95d31d5987363af0acd66d7ee0ed7d20e2eb64a7806b2e02",
  "source_final_reports": {
    "E13": {
      "accepted_edit_count": 624,
      "applicable_patch_count": 610,
      "path": "results/reports/E13_7f4de67853de_1784618186.json",
      "resolved_count": 148,
      "run_count": 1440,
      "sha256": "98cc58a125b5aa5c69e90b46134d9e01a93bd37d7d4f5d2160e0921746fb29a8"
    },
    "E15": {
      "accepted_edit_count": 206,
      "applicable_patch_count": 199,
      "path": "results/reports/E15_7f4de67853de_1784664649.json",
      "resolved_count": 0,
      "run_count": 540,
      "sha256": "842cb293fd4269de8fafdb97b343d3baf78c789a54b94e8b3136d75216e83dab"
    }
  },
  "source_raw_sha256": {
    "E13": "723a554fdb2596f0f8ece3d86f3f9b4333d2589bdae6e638b40798cb3b4c775e",
    "E15": "30d40d4d371a8dac7554648f0066d85660d5486b3db8de686f7b66911d9f6e28"
  },
  "study": "Study 5 end-to-end harness behavior"
}