{
  "by_runner_family": {
    "quantized_vllm": {
      "json_multistep": {
        "n": 5,
        "pass_rate": 0.4,
        "pass_cols": [
          "schema_ok",
          "checks_consistent_ok",
          "stop_semantics_ok",
          "oracle_equiv_ok"
        ],
        "averages": {
          "bucket_score": 1.8
        },
        "top_failures": [
          {
            "reason": "checks_consistent_ok+oracle_equiv_ok",
            "count": 2
          },
          {
            "reason": "checks_consistent_ok+stop_semantics_ok",
            "count": 1
          }
        ],
        "signal_rates": {
          "schema_ok": 1.0,
          "checks_consistent_ok": 0.4,
          "stop_semantics_ok": 0.8,
          "oracle_equiv_ok": 0.6
        },
        "signal_counts": {
          "schema_ok": {
            "present": 5,
            "ok": 5,
            "fail": 0
          },
          "checks_consistent_ok": {
            "present": 5,
            "ok": 2,
            "fail": 3
          },
          "stop_semantics_ok": {
            "present": 5,
            "ok": 4,
            "fail": 1
          },
          "oracle_equiv_ok": {
            "present": 5,
            "ok": 3,
            "fail": 2
          }
        },
        "tier2_rates": {
          "final_consistent_ok": 0.0,
          "final_match_reported": 0.0
        },
        "tier2_counts": {
          "final_consistent_ok": {
            "present": 5,
            "ok": 0,
            "fail": 5
          },
          "final_match_reported": {
            "present": 5,
            "ok": 0,
            "fail": 5
          }
        }
      },
      "stateful_followup": {
        "n": 2,
        "pass_rate": 1.0,
        "pass_cols": [
          "turn1_parse_ok",
          "turn2_parse_ok",
          "turn1_exact_match",
          "turn2_exact_match"
        ],
        "averages": {
          "bucket_score": 2.0
        },
        "top_failures": [],
        "signal_rates": {
          "turn1_parse_ok": 1.0,
          "turn2_parse_ok": 1.0,
          "turn1_exact_match": 1.0,
          "turn2_exact_match": 1.0
        },
        "signal_counts": {
          "turn1_parse_ok": {
            "present": 2,
            "ok": 2,
            "fail": 0
          },
          "turn2_parse_ok": {
            "present": 2,
            "ok": 2,
            "fail": 0
          },
          "turn1_exact_match": {
            "present": 2,
            "ok": 2,
            "fail": 0
          },
          "turn2_exact_match": {
            "present": 2,
            "ok": 2,
            "fail": 0
          }
        },
        "tier2_rates": {},
        "tier2_counts": {}
      },
      "toolcall_only": {
        "n": 2,
        "pass_rate": 1.0,
        "pass_cols": [
          "tool_name_ok",
          "args_ok"
        ],
        "averages": {
          "bucket_score": 2.0
        },
        "top_failures": [],
        "signal_rates": {
          "tool_name_ok": 1.0,
          "args_ok": 1.0
        },
        "signal_counts": {
          "tool_name_ok": {
            "present": 2,
            "ok": 2,
            "fail": 0
          },
          "args_ok": {
            "present": 2,
            "ok": 2,
            "fail": 0
          }
        },
        "tier2_rates": {},
        "tier2_counts": {}
      },
      "mixed_brief_json": {
        "n": 2,
        "pass_rate": 1.0,
        "pass_cols": [
          "answer_line_ok",
          "json_parse_ok",
          "schema_ok"
        ],
        "averages": {
          "bucket_score": 2.0
        },
        "top_failures": [],
        "signal_rates": {
          "answer_line_ok": 1.0,
          "json_parse_ok": 1.0,
          "schema_ok": 1.0
        },
        "signal_counts": {
          "answer_line_ok": {
            "present": 2,
            "ok": 2,
            "fail": 0
          },
          "json_parse_ok": {
            "present": 2,
            "ok": 2,
            "fail": 0
          },
          "schema_ok": {
            "present": 2,
            "ok": 2,
            "fail": 0
          }
        },
        "tier2_rates": {},
        "tier2_counts": {}
      },
      "json": {
        "n": 4,
        "pass_rate": null,
        "pass_cols": [
          "schema_ok",
          "constraints_ok"
        ],
        "averages": {
          "bucket_score": 10.0
        },
        "top_failures": [],
        "signal_rates": {},
        "signal_counts": {},
        "tier2_rates": {},
        "tier2_counts": {}
      },
      "fuzz": {
        "n": 20,
        "pass_rate": null,
        "pass_cols": [
          "schema_ok",
          "constraints_ok"
        ],
        "averages": {
          "bucket_score": 10.0
        },
        "top_failures": [],
        "signal_rates": {},
        "signal_counts": {},
        "tier2_rates": {},
        "tier2_counts": {}
      },
      "toolcall": {
        "n": 2,
        "pass_rate": 1.0,
        "pass_cols": [
          "stage1_tool_parse_ok",
          "stage1_tool_schema_ok"
        ],
        "averages": {
          "bucket_score": 11.0
        },
        "top_failures": [],
        "signal_rates": {
          "stage1_tool_parse_ok": 1.0,
          "stage1_tool_schema_ok": 1.0
        },
        "signal_counts": {
          "stage1_tool_parse_ok": {
            "present": 2,
            "ok": 2,
            "fail": 0
          },
          "stage1_tool_schema_ok": {
            "present": 2,
            "ok": 2,
            "fail": 0
          }
        },
        "tier2_rates": {},
        "tier2_counts": {}
      },
      "mcq": {
        "n": 5,
        "pass_rate": null,
        "pass_cols": [
          "choice_ok"
        ],
        "averages": {
          "bucket_score": 0.6
        },
        "top_failures": [],
        "signal_rates": {},
        "signal_counts": {},
        "tier2_rates": {},
        "tier2_counts": {}
      }
    }
  },
  "generated_from": {
    "rows": 42
  },
  "version_tag": "v7.21",
  "run_id": "20260629_184555"
}