{
  "scope": "Synthetic current-prefix acceptable-action accuracy; not end-to-end agent utility",
  "human_gold": false,
  "validation": {
    "lexical128": {
      "n": 64,
      "correct": 43,
      "acceptable_accuracy": 0.671875,
      "false_abstain": 19,
      "false_abstain_rate": 0.8260869565217391,
      "missed_abstain": 1,
      "missed_abstain_rate": 0.024390243902439025
    },
    "lexical32": {
      "n": 64,
      "correct": 41,
      "acceptable_accuracy": 0.640625,
      "false_abstain": 20,
      "false_abstain_rate": 0.8695652173913043,
      "missed_abstain": 3,
      "missed_abstain_rate": 0.07317073170731707
    },
    "lexical64": {
      "n": 64,
      "correct": 37,
      "acceptable_accuracy": 0.578125,
      "false_abstain": 20,
      "false_abstain_rate": 0.8695652173913043,
      "missed_abstain": 7,
      "missed_abstain_rate": 0.17073170731707318
    },
    "semantic128": {
      "n": 64,
      "correct": 55,
      "acceptable_accuracy": 0.859375,
      "false_abstain": 6,
      "false_abstain_rate": 0.2608695652173913,
      "missed_abstain": 1,
      "missed_abstain_rate": 0.024390243902439025
    }
  },
  "fresh_test_paired_all64": {
    "baseline_correct": 54,
    "candidate_correct": 44,
    "baseline_accuracy": 0.84375,
    "candidate_accuracy": 0.6875,
    "accuracy_difference": -0.15625,
    "paired_bootstrap95": [
      -0.25,
      -0.078125
    ],
    "bootstrap_replicates": 10000,
    "bootstrap_seed": 101,
    "gained_correct_worlds": 0,
    "lost_correct_worlds": 10,
    "select_denominator": 20,
    "baseline_false_abstain": 5,
    "candidate_false_abstain": 5,
    "baseline_false_abstain_rate": 0.25,
    "candidate_false_abstain_rate": 0.25,
    "abstain_denominator": 44,
    "baseline_missed_abstain": 3,
    "candidate_missed_abstain": 10,
    "baseline_missed_abstain_rate": 0.06818181818181818,
    "candidate_missed_abstain_rate": 0.22727272727272727,
    "baseline_conditional_tool_errors": 4,
    "candidate_conditional_tool_errors": 5,
    "baseline_mean_joint_target_set_nll": 0.9775434669867754,
    "candidate_mean_joint_target_set_nll": 3.0828047417095714,
    "additional_false_abstain_worlds": [
      "gec1_dw010"
    ]
  },
  "fresh_test_gate_checks": {
    "accuracy_at_least_80_percent": false,
    "false_abstain_at_most_10_percent": false,
    "missed_abstain_at_most_10_percent": false,
    "both_classes_nonzero": true,
    "extra_correct_at_least_4": false,
    "no_added_false_abstain_worlds": false,
    "candidate_gate_passed": false,
    "baseline_also_fails_false_abstain_gate": true
  },
  "test_independence": {
    "archive_committed_before_fresh_labels": true,
    "labels_committed_before_predictions": true,
    "first_use_of_reserved_fresh64": true,
    "raw_author": "Existing gpt-6-astra, unchanged role",
    "source_adapter": "gpt-6.1-sol high, source only",
    "expert": "gpt-6-astra medium, fresh context, individual current-prefix manual labels",
    "expert_prediction_access": false,
    "human_gold": false,
    "PASS": 64,
    "HOLD": 0,
    "excluded_worlds": 0,
    "semantic_relabels": 0,
    "threshold_searches": 0,
    "replays": 0
  },
  "published_weights": "semantic128 (previous reference; not rejected candidate384)",
  "published_weights_sha256": "645a82c68576b207723a21ee413edbc9e69980a41520cdcb475bf28455c94887",
  "training": {
    "HOLD": 2,
    "PASS": 126,
    "acceptable_target_coverage": {
      "list_directory": 7,
      "read_file": 21,
      "search_content": 30,
      "search_files": 13
    },
    "archive_sha256": "645a82c68576b207723a21ee413edbc9e69980a41520cdcb475bf28455c94887",
    "arm": "semantic128",
    "converged": false,
    "optimizer_receipt_sha256": "9583dd8b0f65c4a388b4e30132d2eaba820e14b2bb9e6ee04e3b02880cb5d5a8",
    "optimizer_stop": "evaluation_budget_exhausted",
    "optimizer_trace_sha256": "ea2462e18f60755348689bc450f3dba96c5768448f1a7a913cedc31a5f93e37c",
    "seconds": 559.4276709999995,
    "target_abstain": 91,
    "target_select": 37,
    "train_count": 128,
    "train_predictions_sha256": "08bbcd8fa0728e48bcc50825049f42b1e770716a8850e3b8d0d73290869da518",
    "training_fingerprint": "33a7f43795a6ad43bbfdb9cf9a2f6602dcb14be3c21c95fee189514ac292c4c8"
  },
  "probabilities": "uncalibrated_softmax; no threshold search",
  "test_policy": "Historical test is public and closed. Do not use it for training, selection, threshold tuning or relabeling and then report independent test results on it.",
  "new_quality_evaluation_or_training_for_release": false
}