| { |
| "schema_version": "toolcall-verifier-classifier-diagnostics/v1", |
| "latest_failed_t4_fast_baseline": { |
| "test_macro_f1": 0.3716486037399564, |
| "test_valid_recall": 0.5279547062986554, |
| "test_wrong_tool_semantic_precision": 0.3870967741935484 |
| }, |
| "validation_gate_metrics": { |
| "rows": 11293, |
| "prediction_column": "pred_label", |
| "confidence_column": "confidence", |
| "false_objection_threshold": 0.9, |
| "valid_rows": 3928, |
| "valid_recall": 0.9814154786150713, |
| "valid_false_objection_at_0_90": { |
| "false_objections": 18, |
| "valid_rows": 3928, |
| "rate": 0.004582484725050916 |
| }, |
| "wrong_tool_semantic_precision": 0.9868630573248408, |
| "wrong_tool_semantic_predicted_rows": 2512, |
| "wrong_arguments_semantic_precision": 0.9890520694259012, |
| "wrong_arguments_semantic_predicted_rows": 3745, |
| "valid_to_wrong_arguments_semantic": { |
| "rows": 37, |
| "valid_rows": 3928, |
| "rate": 0.009419551934826884 |
| }, |
| "wrong_tool_to_wrong_arguments_semantic": { |
| "rows": 1, |
| "wrong_tool_rows": 2540, |
| "rate": 0.0003937007874015748 |
| }, |
| "needs_clarification_support": 41 |
| }, |
| "test_gate_metrics": { |
| "rows": 22370, |
| "prediction_column": "pred_label", |
| "confidence_column": "confidence", |
| "false_objection_threshold": 0.9, |
| "valid_rows": 7836, |
| "valid_recall": 0.9823889739663093, |
| "valid_false_objection_at_0_90": { |
| "false_objections": 52, |
| "valid_rows": 7836, |
| "rate": 0.006636038795303726 |
| }, |
| "wrong_tool_semantic_precision": 0.98904052936311, |
| "wrong_tool_semantic_predicted_rows": 4836, |
| "wrong_arguments_semantic_precision": 0.9878325979409012, |
| "wrong_arguments_semantic_predicted_rows": 7479, |
| "valid_to_wrong_arguments_semantic": { |
| "rows": 81, |
| "valid_rows": 7836, |
| "rate": 0.010336906584992343 |
| }, |
| "wrong_tool_to_wrong_arguments_semantic": { |
| "rows": 4, |
| "wrong_tool_rows": 4921, |
| "rate": 0.000812842918106076 |
| }, |
| "needs_clarification_support": 101 |
| }, |
| "validation_deployment_gate_metrics": { |
| "rows": 11293, |
| "prediction_column": "deployment_pred_label", |
| "confidence_column": "deployment_confidence", |
| "false_objection_threshold": null, |
| "valid_rows": 3928, |
| "valid_recall": 0.9989816700610998, |
| "valid_false_objection_deployment": { |
| "false_objections": 4, |
| "valid_rows": 3928, |
| "rate": 0.0010183299389002036 |
| }, |
| "wrong_tool_semantic_precision": 1.0, |
| "wrong_tool_semantic_predicted_rows": 2416, |
| "wrong_arguments_semantic_precision": 0.9988823693769209, |
| "wrong_arguments_semantic_predicted_rows": 3579, |
| "valid_to_wrong_arguments_semantic": { |
| "rows": 4, |
| "valid_rows": 3928, |
| "rate": 0.0010183299389002036 |
| }, |
| "wrong_tool_to_wrong_arguments_semantic": { |
| "rows": 0, |
| "wrong_tool_rows": 2540, |
| "rate": 0.0 |
| }, |
| "needs_clarification_support": 41 |
| }, |
| "test_deployment_gate_metrics": { |
| "rows": 22370, |
| "prediction_column": "deployment_pred_label", |
| "confidence_column": "deployment_confidence", |
| "false_objection_threshold": null, |
| "valid_rows": 7836, |
| "valid_recall": 0.9961715160796325, |
| "valid_false_objection_deployment": { |
| "false_objections": 30, |
| "valid_rows": 7836, |
| "rate": 0.0038284839203675345 |
| }, |
| "wrong_tool_semantic_precision": 0.998502353444587, |
| "wrong_tool_semantic_predicted_rows": 4674, |
| "wrong_arguments_semantic_precision": 0.9967890548652799, |
| "wrong_arguments_semantic_predicted_rows": 7163, |
| "valid_to_wrong_arguments_semantic": { |
| "rows": 23, |
| "valid_rows": 7836, |
| "rate": 0.00293517100561511 |
| }, |
| "wrong_tool_to_wrong_arguments_semantic": { |
| "rows": 0, |
| "wrong_tool_rows": 4921, |
| "rate": 0.0 |
| }, |
| "needs_clarification_support": 101 |
| }, |
| "validation_confusion_pairs": [ |
| { |
| "true_label": "wrong_arguments_semantic", |
| "pred_label": "valid", |
| "source": "Salesforce/xlam-function-calling-60k", |
| "generator": "unknown", |
| "rows": 62 |
| }, |
| { |
| "true_label": "wrong_tool_semantic", |
| "pred_label": "valid", |
| "source": "Salesforce/xlam-function-calling-60k", |
| "generator": "unknown", |
| "rows": 40 |
| }, |
| { |
| "true_label": "valid", |
| "pred_label": "wrong_arguments_semantic", |
| "source": "Salesforce/xlam-function-calling-60k", |
| "generator": "unknown", |
| "rows": 31 |
| }, |
| { |
| "true_label": "valid", |
| "pred_label": "wrong_tool_semantic", |
| "source": "Team-ACE/ToolACE", |
| "generator": "unknown", |
| "rows": 19 |
| }, |
| { |
| "true_label": "wrong_tool_semantic", |
| "pred_label": "valid", |
| "source": "Team-ACE/ToolACE", |
| "generator": "unknown", |
| "rows": 18 |
| }, |
| { |
| "true_label": "needs_clarification", |
| "pred_label": "valid", |
| "source": "Salesforce/xlam-function-calling-60k", |
| "generator": "unknown", |
| "rows": 9 |
| }, |
| { |
| "true_label": "valid", |
| "pred_label": "wrong_tool_semantic", |
| "source": "Salesforce/xlam-function-calling-60k", |
| "generator": "unknown", |
| "rows": 8 |
| }, |
| { |
| "true_label": "wrong_arguments_semantic", |
| "pred_label": "valid", |
| "source": "Team-ACE/ToolACE", |
| "generator": "unknown", |
| "rows": 6 |
| }, |
| { |
| "true_label": "valid", |
| "pred_label": "wrong_arguments_semantic", |
| "source": "Team-ACE/ToolACE", |
| "generator": "unknown", |
| "rows": 5 |
| }, |
| { |
| "true_label": "wrong_arguments_semantic", |
| "pred_label": "valid", |
| "source": "forge_fixed_width_numeric", |
| "generator": "unknown", |
| "rows": 5 |
| }, |
| { |
| "true_label": "wrong_arguments_semantic", |
| "pred_label": "valid", |
| "source": "glaiveai/glaive-function-calling-v2", |
| "generator": "unknown", |
| "rows": 4 |
| }, |
| { |
| "true_label": "valid", |
| "pred_label": "tool_not_needed", |
| "source": "agent_training_hf", |
| "generator": "unknown", |
| "rows": 4 |
| }, |
| { |
| "true_label": "tool_not_needed", |
| "pred_label": "wrong_tool_semantic", |
| "source": "agent_training_hf", |
| "generator": "unknown", |
| "rows": 4 |
| }, |
| { |
| "true_label": "needs_clarification", |
| "pred_label": "valid", |
| "source": "Team-ACE/ToolACE", |
| "generator": "unknown", |
| "rows": 3 |
| }, |
| { |
| "true_label": "needs_clarification", |
| "pred_label": "wrong_arguments_semantic", |
| "source": "Salesforce/xlam-function-calling-60k", |
| "generator": "unknown", |
| "rows": 3 |
| }, |
| { |
| "true_label": "valid", |
| "pred_label": "needs_clarification", |
| "source": "Salesforce/xlam-function-calling-60k", |
| "generator": "unknown", |
| "rows": 3 |
| }, |
| { |
| "true_label": "wrong_arguments_semantic", |
| "pred_label": "wrong_tool_semantic", |
| "source": "Team-ACE/ToolACE", |
| "generator": "unknown", |
| "rows": 2 |
| }, |
| { |
| "true_label": "wrong_arguments_semantic", |
| "pred_label": "valid", |
| "source": "agent_training_hf", |
| "generator": "unknown", |
| "rows": 2 |
| }, |
| { |
| "true_label": "valid", |
| "pred_label": "needs_clarification", |
| "source": "glaiveai/glaive-function-calling-v2", |
| "generator": "unknown", |
| "rows": 1 |
| }, |
| { |
| "true_label": "needs_clarification", |
| "pred_label": "valid", |
| "source": "glaiveai/glaive-function-calling-v2", |
| "generator": "unknown", |
| "rows": 1 |
| }, |
| { |
| "true_label": "valid", |
| "pred_label": "needs_clarification", |
| "source": "Team-ACE/ToolACE", |
| "generator": "unknown", |
| "rows": 1 |
| }, |
| { |
| "true_label": "valid", |
| "pred_label": "wrong_arguments_semantic", |
| "source": "glaiveai/glaive-function-calling-v2", |
| "generator": "unknown", |
| "rows": 1 |
| }, |
| { |
| "true_label": "wrong_tool_semantic", |
| "pred_label": "tool_not_needed", |
| "source": "agent_training_hf", |
| "generator": "unknown", |
| "rows": 1 |
| }, |
| { |
| "true_label": "wrong_tool_semantic", |
| "pred_label": "valid", |
| "source": "glaiveai/glaive-function-calling-v2", |
| "generator": "unknown", |
| "rows": 1 |
| }, |
| { |
| "true_label": "wrong_tool_semantic", |
| "pred_label": "wrong_arguments_semantic", |
| "source": "Team-ACE/ToolACE", |
| "generator": "unknown", |
| "rows": 1 |
| } |
| ], |
| "test_confusion_pairs": [ |
| { |
| "true_label": "wrong_arguments_semantic", |
| "pred_label": "valid", |
| "source": "Salesforce/xlam-function-calling-60k", |
| "generator": "unknown", |
| "rows": 126 |
| }, |
| { |
| "true_label": "wrong_tool_semantic", |
| "pred_label": "valid", |
| "source": "Salesforce/xlam-function-calling-60k", |
| "generator": "unknown", |
| "rows": 73 |
| }, |
| { |
| "true_label": "valid", |
| "pred_label": "wrong_arguments_semantic", |
| "source": "Salesforce/xlam-function-calling-60k", |
| "generator": "unknown", |
| "rows": 60 |
| }, |
| { |
| "true_label": "wrong_tool_semantic", |
| "pred_label": "valid", |
| "source": "Team-ACE/ToolACE", |
| "generator": "unknown", |
| "rows": 58 |
| }, |
| { |
| "true_label": "needs_clarification", |
| "pred_label": "valid", |
| "source": "Salesforce/xlam-function-calling-60k", |
| "generator": "unknown", |
| "rows": 30 |
| }, |
| { |
| "true_label": "valid", |
| "pred_label": "wrong_tool_semantic", |
| "source": "Team-ACE/ToolACE", |
| "generator": "unknown", |
| "rows": 29 |
| }, |
| { |
| "true_label": "valid", |
| "pred_label": "wrong_tool_semantic", |
| "source": "Salesforce/xlam-function-calling-60k", |
| "generator": "unknown", |
| "rows": 18 |
| }, |
| { |
| "true_label": "valid", |
| "pred_label": "wrong_arguments_semantic", |
| "source": "Team-ACE/ToolACE", |
| "generator": "unknown", |
| "rows": 16 |
| }, |
| { |
| "true_label": "wrong_arguments_semantic", |
| "pred_label": "valid", |
| "source": "Team-ACE/ToolACE", |
| "generator": "unknown", |
| "rows": 15 |
| }, |
| { |
| "true_label": "needs_clarification", |
| "pred_label": "valid", |
| "source": "Team-ACE/ToolACE", |
| "generator": "unknown", |
| "rows": 11 |
| }, |
| { |
| "true_label": "needs_clarification", |
| "pred_label": "valid", |
| "source": "glaiveai/glaive-function-calling-v2", |
| "generator": "unknown", |
| "rows": 9 |
| }, |
| { |
| "true_label": "needs_clarification", |
| "pred_label": "wrong_arguments_semantic", |
| "source": "Salesforce/xlam-function-calling-60k", |
| "generator": "unknown", |
| "rows": 5 |
| }, |
| { |
| "true_label": "wrong_arguments_semantic", |
| "pred_label": "valid", |
| "source": "glaiveai/glaive-function-calling-v2", |
| "generator": "unknown", |
| "rows": 5 |
| }, |
| { |
| "true_label": "valid", |
| "pred_label": "wrong_arguments_semantic", |
| "source": "glaiveai/glaive-function-calling-v2", |
| "generator": "unknown", |
| "rows": 4 |
| }, |
| { |
| "true_label": "wrong_arguments_semantic", |
| "pred_label": "wrong_tool_semantic", |
| "source": "Salesforce/xlam-function-calling-60k", |
| "generator": "unknown", |
| "rows": 2 |
| }, |
| { |
| "true_label": "valid", |
| "pred_label": "tool_not_needed", |
| "source": "agent_training_hf", |
| "generator": "unknown", |
| "rows": 2 |
| }, |
| { |
| "true_label": "valid", |
| "pred_label": "needs_clarification", |
| "source": "Salesforce/xlam-function-calling-60k", |
| "generator": "unknown", |
| "rows": 2 |
| }, |
| { |
| "true_label": "tool_not_needed", |
| "pred_label": "wrong_tool_semantic", |
| "source": "agent_training_hf", |
| "generator": "unknown", |
| "rows": 2 |
| }, |
| { |
| "true_label": "valid", |
| "pred_label": "needs_clarification", |
| "source": "Team-ACE/ToolACE", |
| "generator": "unknown", |
| "rows": 2 |
| }, |
| { |
| "true_label": "valid", |
| "pred_label": "needs_clarification", |
| "source": "glaiveai/glaive-function-calling-v2", |
| "generator": "unknown", |
| "rows": 2 |
| }, |
| { |
| "true_label": "wrong_tool_semantic", |
| "pred_label": "wrong_arguments_semantic", |
| "source": "Team-ACE/ToolACE", |
| "generator": "unknown", |
| "rows": 2 |
| }, |
| { |
| "true_label": "wrong_tool_semantic", |
| "pred_label": "valid", |
| "source": "glaiveai/glaive-function-calling-v2", |
| "generator": "unknown", |
| "rows": 2 |
| }, |
| { |
| "true_label": "wrong_arguments_semantic", |
| "pred_label": "needs_clarification", |
| "source": "Salesforce/xlam-function-calling-60k", |
| "generator": "unknown", |
| "rows": 2 |
| }, |
| { |
| "true_label": "wrong_tool_semantic", |
| "pred_label": "wrong_arguments_semantic", |
| "source": "Salesforce/xlam-function-calling-60k", |
| "generator": "unknown", |
| "rows": 2 |
| }, |
| { |
| "true_label": "tool_not_needed", |
| "pred_label": "valid", |
| "source": "agent_training_hf", |
| "generator": "unknown", |
| "rows": 1 |
| }, |
| { |
| "true_label": "needs_clarification", |
| "pred_label": "wrong_tool_semantic", |
| "source": "Team-ACE/ToolACE", |
| "generator": "unknown", |
| "rows": 1 |
| }, |
| { |
| "true_label": "needs_clarification", |
| "pred_label": "wrong_arguments_semantic", |
| "source": "Team-ACE/ToolACE", |
| "generator": "unknown", |
| "rows": 1 |
| }, |
| { |
| "true_label": "valid", |
| "pred_label": "tool_not_needed", |
| "source": "Team-ACE/ToolACE", |
| "generator": "unknown", |
| "rows": 1 |
| }, |
| { |
| "true_label": "valid", |
| "pred_label": "wrong_tool_semantic", |
| "source": "glaiveai/glaive-function-calling-v2", |
| "generator": "unknown", |
| "rows": 1 |
| }, |
| { |
| "true_label": "valid", |
| "pred_label": "wrong_arguments_semantic", |
| "source": "forge_trace", |
| "generator": "unknown", |
| "rows": 1 |
| }, |
| { |
| "true_label": "wrong_arguments_semantic", |
| "pred_label": "valid", |
| "source": "forge_fixed_width_numeric", |
| "generator": "unknown", |
| "rows": 1 |
| }, |
| { |
| "true_label": "wrong_arguments_semantic", |
| "pred_label": "tool_not_needed", |
| "source": "agent_training_hf", |
| "generator": "unknown", |
| "rows": 1 |
| }, |
| { |
| "true_label": "wrong_arguments_semantic", |
| "pred_label": "tool_not_needed", |
| "source": "Team-ACE/ToolACE", |
| "generator": "unknown", |
| "rows": 1 |
| }, |
| { |
| "true_label": "wrong_arguments_semantic", |
| "pred_label": "valid", |
| "source": "forge_error_recovery_numeric", |
| "generator": "unknown", |
| "rows": 1 |
| }, |
| { |
| "true_label": "wrong_arguments_semantic", |
| "pred_label": "valid", |
| "source": "agent_training_hf", |
| "generator": "unknown", |
| "rows": 1 |
| }, |
| { |
| "true_label": "wrong_tool_semantic", |
| "pred_label": "tool_not_needed", |
| "source": "Team-ACE/ToolACE", |
| "generator": "unknown", |
| "rows": 1 |
| } |
| ] |
| } |
