| { |
| "by_runner_family": { |
| "quantized_vllm": { |
| "json_multistep": { |
| "n": 5, |
| "pass_rate": 0.4, |
| "pass_cols": [ |
| "schema_ok", |
| "checks_consistent_ok", |
| "stop_semantics_ok", |
| "oracle_equiv_ok" |
| ], |
| "averages": { |
| "bucket_score": 1.8 |
| }, |
| "top_failures": [ |
| { |
| "reason": "checks_consistent_ok+oracle_equiv_ok", |
| "count": 2 |
| }, |
| { |
| "reason": "checks_consistent_ok+stop_semantics_ok", |
| "count": 1 |
| } |
| ], |
| "signal_rates": { |
| "schema_ok": 1.0, |
| "checks_consistent_ok": 0.4, |
| "stop_semantics_ok": 0.8, |
| "oracle_equiv_ok": 0.6 |
| }, |
| "signal_counts": { |
| "schema_ok": { |
| "present": 5, |
| "ok": 5, |
| "fail": 0 |
| }, |
| "checks_consistent_ok": { |
| "present": 5, |
| "ok": 2, |
| "fail": 3 |
| }, |
| "stop_semantics_ok": { |
| "present": 5, |
| "ok": 4, |
| "fail": 1 |
| }, |
| "oracle_equiv_ok": { |
| "present": 5, |
| "ok": 3, |
| "fail": 2 |
| } |
| }, |
| "tier2_rates": { |
| "final_consistent_ok": 0.0, |
| "final_match_reported": 0.0 |
| }, |
| "tier2_counts": { |
| "final_consistent_ok": { |
| "present": 5, |
| "ok": 0, |
| "fail": 5 |
| }, |
| "final_match_reported": { |
| "present": 5, |
| "ok": 0, |
| "fail": 5 |
| } |
| } |
| }, |
| "stateful_followup": { |
| "n": 2, |
| "pass_rate": 1.0, |
| "pass_cols": [ |
| "turn1_parse_ok", |
| "turn2_parse_ok", |
| "turn1_exact_match", |
| "turn2_exact_match" |
| ], |
| "averages": { |
| "bucket_score": 2.0 |
| }, |
| "top_failures": [], |
| "signal_rates": { |
| "turn1_parse_ok": 1.0, |
| "turn2_parse_ok": 1.0, |
| "turn1_exact_match": 1.0, |
| "turn2_exact_match": 1.0 |
| }, |
| "signal_counts": { |
| "turn1_parse_ok": { |
| "present": 2, |
| "ok": 2, |
| "fail": 0 |
| }, |
| "turn2_parse_ok": { |
| "present": 2, |
| "ok": 2, |
| "fail": 0 |
| }, |
| "turn1_exact_match": { |
| "present": 2, |
| "ok": 2, |
| "fail": 0 |
| }, |
| "turn2_exact_match": { |
| "present": 2, |
| "ok": 2, |
| "fail": 0 |
| } |
| }, |
| "tier2_rates": {}, |
| "tier2_counts": {} |
| }, |
| "toolcall_only": { |
| "n": 2, |
| "pass_rate": 1.0, |
| "pass_cols": [ |
| "tool_name_ok", |
| "args_ok" |
| ], |
| "averages": { |
| "bucket_score": 2.0 |
| }, |
| "top_failures": [], |
| "signal_rates": { |
| "tool_name_ok": 1.0, |
| "args_ok": 1.0 |
| }, |
| "signal_counts": { |
| "tool_name_ok": { |
| "present": 2, |
| "ok": 2, |
| "fail": 0 |
| }, |
| "args_ok": { |
| "present": 2, |
| "ok": 2, |
| "fail": 0 |
| } |
| }, |
| "tier2_rates": {}, |
| "tier2_counts": {} |
| }, |
| "mixed_brief_json": { |
| "n": 2, |
| "pass_rate": 1.0, |
| "pass_cols": [ |
| "answer_line_ok", |
| "json_parse_ok", |
| "schema_ok" |
| ], |
| "averages": { |
| "bucket_score": 2.0 |
| }, |
| "top_failures": [], |
| "signal_rates": { |
| "answer_line_ok": 1.0, |
| "json_parse_ok": 1.0, |
| "schema_ok": 1.0 |
| }, |
| "signal_counts": { |
| "answer_line_ok": { |
| "present": 2, |
| "ok": 2, |
| "fail": 0 |
| }, |
| "json_parse_ok": { |
| "present": 2, |
| "ok": 2, |
| "fail": 0 |
| }, |
| "schema_ok": { |
| "present": 2, |
| "ok": 2, |
| "fail": 0 |
| } |
| }, |
| "tier2_rates": {}, |
| "tier2_counts": {} |
| }, |
| "json": { |
| "n": 4, |
| "pass_rate": null, |
| "pass_cols": [ |
| "schema_ok", |
| "constraints_ok" |
| ], |
| "averages": { |
| "bucket_score": 10.0 |
| }, |
| "top_failures": [], |
| "signal_rates": {}, |
| "signal_counts": {}, |
| "tier2_rates": {}, |
| "tier2_counts": {} |
| }, |
| "fuzz": { |
| "n": 20, |
| "pass_rate": null, |
| "pass_cols": [ |
| "schema_ok", |
| "constraints_ok" |
| ], |
| "averages": { |
| "bucket_score": 10.0 |
| }, |
| "top_failures": [], |
| "signal_rates": {}, |
| "signal_counts": {}, |
| "tier2_rates": {}, |
| "tier2_counts": {} |
| }, |
| "toolcall": { |
| "n": 2, |
| "pass_rate": 1.0, |
| "pass_cols": [ |
| "stage1_tool_parse_ok", |
| "stage1_tool_schema_ok" |
| ], |
| "averages": { |
| "bucket_score": 11.0 |
| }, |
| "top_failures": [], |
| "signal_rates": { |
| "stage1_tool_parse_ok": 1.0, |
| "stage1_tool_schema_ok": 1.0 |
| }, |
| "signal_counts": { |
| "stage1_tool_parse_ok": { |
| "present": 2, |
| "ok": 2, |
| "fail": 0 |
| }, |
| "stage1_tool_schema_ok": { |
| "present": 2, |
| "ok": 2, |
| "fail": 0 |
| } |
| }, |
| "tier2_rates": {}, |
| "tier2_counts": {} |
| }, |
| "mcq": { |
| "n": 5, |
| "pass_rate": null, |
| "pass_cols": [ |
| "choice_ok" |
| ], |
| "averages": { |
| "bucket_score": 0.6 |
| }, |
| "top_failures": [], |
| "signal_rates": {}, |
| "signal_counts": {}, |
| "tier2_rates": {}, |
| "tier2_counts": {} |
| } |
| } |
| }, |
| "generated_from": { |
| "rows": 42 |
| }, |
| "version_tag": "v7.21", |
| "run_id": "20260629_184555" |
| } |
