{
  "schema_version": "toolcall-verifier-classifier-diagnostics/v1",
  "latest_failed_t4_fast_baseline": {
    "test_macro_f1": 0.3716486037399564,
    "test_valid_recall": 0.5279547062986554,
    "test_wrong_tool_semantic_precision": 0.3870967741935484
  },
  "validation_gate_metrics": {
    "rows": 11293,
    "prediction_column": "pred_label",
    "confidence_column": "confidence",
    "false_objection_threshold": 0.9,
    "valid_rows": 3928,
    "valid_recall": 0.9814154786150713,
    "valid_false_objection_at_0_90": {
      "false_objections": 18,
      "valid_rows": 3928,
      "rate": 0.004582484725050916
    },
    "wrong_tool_semantic_precision": 0.9868630573248408,
    "wrong_tool_semantic_predicted_rows": 2512,
    "wrong_arguments_semantic_precision": 0.9890520694259012,
    "wrong_arguments_semantic_predicted_rows": 3745,
    "valid_to_wrong_arguments_semantic": {
      "rows": 37,
      "valid_rows": 3928,
      "rate": 0.009419551934826884
    },
    "wrong_tool_to_wrong_arguments_semantic": {
      "rows": 1,
      "wrong_tool_rows": 2540,
      "rate": 0.0003937007874015748
    },
    "needs_clarification_support": 41
  },
  "test_gate_metrics": {
    "rows": 22370,
    "prediction_column": "pred_label",
    "confidence_column": "confidence",
    "false_objection_threshold": 0.9,
    "valid_rows": 7836,
    "valid_recall": 0.9823889739663093,
    "valid_false_objection_at_0_90": {
      "false_objections": 52,
      "valid_rows": 7836,
      "rate": 0.006636038795303726
    },
    "wrong_tool_semantic_precision": 0.98904052936311,
    "wrong_tool_semantic_predicted_rows": 4836,
    "wrong_arguments_semantic_precision": 0.9878325979409012,
    "wrong_arguments_semantic_predicted_rows": 7479,
    "valid_to_wrong_arguments_semantic": {
      "rows": 81,
      "valid_rows": 7836,
      "rate": 0.010336906584992343
    },
    "wrong_tool_to_wrong_arguments_semantic": {
      "rows": 4,
      "wrong_tool_rows": 4921,
      "rate": 0.000812842918106076
    },
    "needs_clarification_support": 101
  },
  "validation_deployment_gate_metrics": {
    "rows": 11293,
    "prediction_column": "deployment_pred_label",
    "confidence_column": "deployment_confidence",
    "false_objection_threshold": null,
    "valid_rows": 3928,
    "valid_recall": 0.9989816700610998,
    "valid_false_objection_deployment": {
      "false_objections": 4,
      "valid_rows": 3928,
      "rate": 0.0010183299389002036
    },
    "wrong_tool_semantic_precision": 1.0,
    "wrong_tool_semantic_predicted_rows": 2416,
    "wrong_arguments_semantic_precision": 0.9988823693769209,
    "wrong_arguments_semantic_predicted_rows": 3579,
    "valid_to_wrong_arguments_semantic": {
      "rows": 4,
      "valid_rows": 3928,
      "rate": 0.0010183299389002036
    },
    "wrong_tool_to_wrong_arguments_semantic": {
      "rows": 0,
      "wrong_tool_rows": 2540,
      "rate": 0.0
    },
    "needs_clarification_support": 41
  },
  "test_deployment_gate_metrics": {
    "rows": 22370,
    "prediction_column": "deployment_pred_label",
    "confidence_column": "deployment_confidence",
    "false_objection_threshold": null,
    "valid_rows": 7836,
    "valid_recall": 0.9961715160796325,
    "valid_false_objection_deployment": {
      "false_objections": 30,
      "valid_rows": 7836,
      "rate": 0.0038284839203675345
    },
    "wrong_tool_semantic_precision": 0.998502353444587,
    "wrong_tool_semantic_predicted_rows": 4674,
    "wrong_arguments_semantic_precision": 0.9967890548652799,
    "wrong_arguments_semantic_predicted_rows": 7163,
    "valid_to_wrong_arguments_semantic": {
      "rows": 23,
      "valid_rows": 7836,
      "rate": 0.00293517100561511
    },
    "wrong_tool_to_wrong_arguments_semantic": {
      "rows": 0,
      "wrong_tool_rows": 4921,
      "rate": 0.0
    },
    "needs_clarification_support": 101
  },
  "validation_confusion_pairs": [
    {
      "true_label": "wrong_arguments_semantic",
      "pred_label": "valid",
      "source": "Salesforce/xlam-function-calling-60k",
      "generator": "unknown",
      "rows": 62
    },
    {
      "true_label": "wrong_tool_semantic",
      "pred_label": "valid",
      "source": "Salesforce/xlam-function-calling-60k",
      "generator": "unknown",
      "rows": 40
    },
    {
      "true_label": "valid",
      "pred_label": "wrong_arguments_semantic",
      "source": "Salesforce/xlam-function-calling-60k",
      "generator": "unknown",
      "rows": 31
    },
    {
      "true_label": "valid",
      "pred_label": "wrong_tool_semantic",
      "source": "Team-ACE/ToolACE",
      "generator": "unknown",
      "rows": 19
    },
    {
      "true_label": "wrong_tool_semantic",
      "pred_label": "valid",
      "source": "Team-ACE/ToolACE",
      "generator": "unknown",
      "rows": 18
    },
    {
      "true_label": "needs_clarification",
      "pred_label": "valid",
      "source": "Salesforce/xlam-function-calling-60k",
      "generator": "unknown",
      "rows": 9
    },
    {
      "true_label": "valid",
      "pred_label": "wrong_tool_semantic",
      "source": "Salesforce/xlam-function-calling-60k",
      "generator": "unknown",
      "rows": 8
    },
    {
      "true_label": "wrong_arguments_semantic",
      "pred_label": "valid",
      "source": "Team-ACE/ToolACE",
      "generator": "unknown",
      "rows": 6
    },
    {
      "true_label": "valid",
      "pred_label": "wrong_arguments_semantic",
      "source": "Team-ACE/ToolACE",
      "generator": "unknown",
      "rows": 5
    },
    {
      "true_label": "wrong_arguments_semantic",
      "pred_label": "valid",
      "source": "forge_fixed_width_numeric",
      "generator": "unknown",
      "rows": 5
    },
    {
      "true_label": "wrong_arguments_semantic",
      "pred_label": "valid",
      "source": "glaiveai/glaive-function-calling-v2",
      "generator": "unknown",
      "rows": 4
    },
    {
      "true_label": "valid",
      "pred_label": "tool_not_needed",
      "source": "agent_training_hf",
      "generator": "unknown",
      "rows": 4
    },
    {
      "true_label": "tool_not_needed",
      "pred_label": "wrong_tool_semantic",
      "source": "agent_training_hf",
      "generator": "unknown",
      "rows": 4
    },
    {
      "true_label": "needs_clarification",
      "pred_label": "valid",
      "source": "Team-ACE/ToolACE",
      "generator": "unknown",
      "rows": 3
    },
    {
      "true_label": "needs_clarification",
      "pred_label": "wrong_arguments_semantic",
      "source": "Salesforce/xlam-function-calling-60k",
      "generator": "unknown",
      "rows": 3
    },
    {
      "true_label": "valid",
      "pred_label": "needs_clarification",
      "source": "Salesforce/xlam-function-calling-60k",
      "generator": "unknown",
      "rows": 3
    },
    {
      "true_label": "wrong_arguments_semantic",
      "pred_label": "wrong_tool_semantic",
      "source": "Team-ACE/ToolACE",
      "generator": "unknown",
      "rows": 2
    },
    {
      "true_label": "wrong_arguments_semantic",
      "pred_label": "valid",
      "source": "agent_training_hf",
      "generator": "unknown",
      "rows": 2
    },
    {
      "true_label": "valid",
      "pred_label": "needs_clarification",
      "source": "glaiveai/glaive-function-calling-v2",
      "generator": "unknown",
      "rows": 1
    },
    {
      "true_label": "needs_clarification",
      "pred_label": "valid",
      "source": "glaiveai/glaive-function-calling-v2",
      "generator": "unknown",
      "rows": 1
    },
    {
      "true_label": "valid",
      "pred_label": "needs_clarification",
      "source": "Team-ACE/ToolACE",
      "generator": "unknown",
      "rows": 1
    },
    {
      "true_label": "valid",
      "pred_label": "wrong_arguments_semantic",
      "source": "glaiveai/glaive-function-calling-v2",
      "generator": "unknown",
      "rows": 1
    },
    {
      "true_label": "wrong_tool_semantic",
      "pred_label": "tool_not_needed",
      "source": "agent_training_hf",
      "generator": "unknown",
      "rows": 1
    },
    {
      "true_label": "wrong_tool_semantic",
      "pred_label": "valid",
      "source": "glaiveai/glaive-function-calling-v2",
      "generator": "unknown",
      "rows": 1
    },
    {
      "true_label": "wrong_tool_semantic",
      "pred_label": "wrong_arguments_semantic",
      "source": "Team-ACE/ToolACE",
      "generator": "unknown",
      "rows": 1
    }
  ],
  "test_confusion_pairs": [
    {
      "true_label": "wrong_arguments_semantic",
      "pred_label": "valid",
      "source": "Salesforce/xlam-function-calling-60k",
      "generator": "unknown",
      "rows": 126
    },
    {
      "true_label": "wrong_tool_semantic",
      "pred_label": "valid",
      "source": "Salesforce/xlam-function-calling-60k",
      "generator": "unknown",
      "rows": 73
    },
    {
      "true_label": "valid",
      "pred_label": "wrong_arguments_semantic",
      "source": "Salesforce/xlam-function-calling-60k",
      "generator": "unknown",
      "rows": 60
    },
    {
      "true_label": "wrong_tool_semantic",
      "pred_label": "valid",
      "source": "Team-ACE/ToolACE",
      "generator": "unknown",
      "rows": 58
    },
    {
      "true_label": "needs_clarification",
      "pred_label": "valid",
      "source": "Salesforce/xlam-function-calling-60k",
      "generator": "unknown",
      "rows": 30
    },
    {
      "true_label": "valid",
      "pred_label": "wrong_tool_semantic",
      "source": "Team-ACE/ToolACE",
      "generator": "unknown",
      "rows": 29
    },
    {
      "true_label": "valid",
      "pred_label": "wrong_tool_semantic",
      "source": "Salesforce/xlam-function-calling-60k",
      "generator": "unknown",
      "rows": 18
    },
    {
      "true_label": "valid",
      "pred_label": "wrong_arguments_semantic",
      "source": "Team-ACE/ToolACE",
      "generator": "unknown",
      "rows": 16
    },
    {
      "true_label": "wrong_arguments_semantic",
      "pred_label": "valid",
      "source": "Team-ACE/ToolACE",
      "generator": "unknown",
      "rows": 15
    },
    {
      "true_label": "needs_clarification",
      "pred_label": "valid",
      "source": "Team-ACE/ToolACE",
      "generator": "unknown",
      "rows": 11
    },
    {
      "true_label": "needs_clarification",
      "pred_label": "valid",
      "source": "glaiveai/glaive-function-calling-v2",
      "generator": "unknown",
      "rows": 9
    },
    {
      "true_label": "needs_clarification",
      "pred_label": "wrong_arguments_semantic",
      "source": "Salesforce/xlam-function-calling-60k",
      "generator": "unknown",
      "rows": 5
    },
    {
      "true_label": "wrong_arguments_semantic",
      "pred_label": "valid",
      "source": "glaiveai/glaive-function-calling-v2",
      "generator": "unknown",
      "rows": 5
    },
    {
      "true_label": "valid",
      "pred_label": "wrong_arguments_semantic",
      "source": "glaiveai/glaive-function-calling-v2",
      "generator": "unknown",
      "rows": 4
    },
    {
      "true_label": "wrong_arguments_semantic",
      "pred_label": "wrong_tool_semantic",
      "source": "Salesforce/xlam-function-calling-60k",
      "generator": "unknown",
      "rows": 2
    },
    {
      "true_label": "valid",
      "pred_label": "tool_not_needed",
      "source": "agent_training_hf",
      "generator": "unknown",
      "rows": 2
    },
    {
      "true_label": "valid",
      "pred_label": "needs_clarification",
      "source": "Salesforce/xlam-function-calling-60k",
      "generator": "unknown",
      "rows": 2
    },
    {
      "true_label": "tool_not_needed",
      "pred_label": "wrong_tool_semantic",
      "source": "agent_training_hf",
      "generator": "unknown",
      "rows": 2
    },
    {
      "true_label": "valid",
      "pred_label": "needs_clarification",
      "source": "Team-ACE/ToolACE",
      "generator": "unknown",
      "rows": 2
    },
    {
      "true_label": "valid",
      "pred_label": "needs_clarification",
      "source": "glaiveai/glaive-function-calling-v2",
      "generator": "unknown",
      "rows": 2
    },
    {
      "true_label": "wrong_tool_semantic",
      "pred_label": "wrong_arguments_semantic",
      "source": "Team-ACE/ToolACE",
      "generator": "unknown",
      "rows": 2
    },
    {
      "true_label": "wrong_tool_semantic",
      "pred_label": "valid",
      "source": "glaiveai/glaive-function-calling-v2",
      "generator": "unknown",
      "rows": 2
    },
    {
      "true_label": "wrong_arguments_semantic",
      "pred_label": "needs_clarification",
      "source": "Salesforce/xlam-function-calling-60k",
      "generator": "unknown",
      "rows": 2
    },
    {
      "true_label": "wrong_tool_semantic",
      "pred_label": "wrong_arguments_semantic",
      "source": "Salesforce/xlam-function-calling-60k",
      "generator": "unknown",
      "rows": 2
    },
    {
      "true_label": "tool_not_needed",
      "pred_label": "valid",
      "source": "agent_training_hf",
      "generator": "unknown",
      "rows": 1
    },
    {
      "true_label": "needs_clarification",
      "pred_label": "wrong_tool_semantic",
      "source": "Team-ACE/ToolACE",
      "generator": "unknown",
      "rows": 1
    },
    {
      "true_label": "needs_clarification",
      "pred_label": "wrong_arguments_semantic",
      "source": "Team-ACE/ToolACE",
      "generator": "unknown",
      "rows": 1
    },
    {
      "true_label": "valid",
      "pred_label": "tool_not_needed",
      "source": "Team-ACE/ToolACE",
      "generator": "unknown",
      "rows": 1
    },
    {
      "true_label": "valid",
      "pred_label": "wrong_tool_semantic",
      "source": "glaiveai/glaive-function-calling-v2",
      "generator": "unknown",
      "rows": 1
    },
    {
      "true_label": "valid",
      "pred_label": "wrong_arguments_semantic",
      "source": "forge_trace",
      "generator": "unknown",
      "rows": 1
    },
    {
      "true_label": "wrong_arguments_semantic",
      "pred_label": "valid",
      "source": "forge_fixed_width_numeric",
      "generator": "unknown",
      "rows": 1
    },
    {
      "true_label": "wrong_arguments_semantic",
      "pred_label": "tool_not_needed",
      "source": "agent_training_hf",
      "generator": "unknown",
      "rows": 1
    },
    {
      "true_label": "wrong_arguments_semantic",
      "pred_label": "tool_not_needed",
      "source": "Team-ACE/ToolACE",
      "generator": "unknown",
      "rows": 1
    },
    {
      "true_label": "wrong_arguments_semantic",
      "pred_label": "valid",
      "source": "forge_error_recovery_numeric",
      "generator": "unknown",
      "rows": 1
    },
    {
      "true_label": "wrong_arguments_semantic",
      "pred_label": "valid",
      "source": "agent_training_hf",
      "generator": "unknown",
      "rows": 1
    },
    {
      "true_label": "wrong_tool_semantic",
      "pred_label": "tool_not_needed",
      "source": "Team-ACE/ToolACE",
      "generator": "unknown",
      "rows": 1
    }
  ]
}