{
  "schema_version": "toolcall-verifier-telemetry-diagnostics/v1",
  "telemetry_only": true,
  "used_for_training": false,
  "notes": [
    "Use these rows to diagnose false objections and dataset gaps.",
    "Do not treat top-k telemetry from the current artifact as promotion evidence."
  ],
  "files": [
    {
      "path": "/content/proxy_classifier_budget_8192.jsonl",
      "kind": "proxy_classifier",
      "rows": 1564,
      "predicted_labels": {
        "wrong_tool_semantic": 1292,
        "valid": 188,
        "wrong_arguments_semantic": 84
      },
      "actions": {
        "shadow_only": 1376,
        "allow": 188
      },
      "terminal_or_terminal_like": {
        "rows": 405,
        "predicted_labels": {
          "wrong_tool_semantic": 343,
          "valid": 38,
          "wrong_arguments_semantic": 24
        },
        "tools": {
          "get_skill_summary": 100,
          "summarize": 59,
          "respond": 54,
          "submit_report": 40,
          "report": 30,
          "submit_hiring_decision": 22,
          "present": 20,
          "recommend": 20,
          "diagnose": 20,
          "submit_audit_report": 20,
          "legacy_submit_audit": 20
        }
      },
      "numeric_string_candidates": {
        "rows": 51,
        "predicted_labels": {
          "wrong_tool_semantic": 50,
          "valid": 1
        },
        "tools": {
          "fetch": 50,
          "submit_audit_report": 1
        }
      },
      "fixed_width_numeric_candidates": {
        "rows": 30,
        "predicted_labels": {
          "wrong_tool_semantic": 30
        },
        "tools": {
          "fetch": 30
        }
      },
      "non_valid_top_label_with_close_valid_margin": [
        {
          "tool": "submit_audit_report",
          "label": "wrong_arguments_semantic",
          "confidence": 0.34632304310798645,
          "valid_confidence": 0.33387550711631775,
          "margin": 0.012447535991668701,
          "user_request": "Run our Q4 2024 expense audit. Flag any transaction of $5,000 or more from vendors NOT on our approved list. Submit the audit report with: comma-separated transaction IDs, total flagged amount in USD, and the vendor of the single largest flagged transaction.",
          "candidate_call": {
            "name": "submit_audit_report",
            "arguments": {
              "transaction_ids": "TX-1001,TX-1002,TX-1003,TX-1005,TX-1006,TX-1007,TX-1008,TX-1012",
              "total_flagged_usd": "73160.00",
              "top_vendor": "Wonka Industries"
            }
          }
        },
        {
          "tool": "legacy_aggregate_subtotal",
          "label": "wrong_tool_semantic",
          "confidence": 0.4037249982357025,
          "valid_confidence": 0.3830156922340393,
          "margin": 0.020709306001663208,
          "user_request": "Run a Q4 2024 (Oct 1 - Dec 31) compliance audit on Acme Corp (account ACC-12345). Pull the account balance and Q4 transactions, categorize at least one transaction, run a US-region compliance check for the period, calculate the subtotal of all transaction amounts in USD, and submit the audit report with keys: flagged_count (int), total_usd (str), compliance_status (str), txn_count (int).",
          "candidate_call": {
            "name": "legacy_aggregate_subtotal",
            "arguments": {
              "amounts": "5000.00|12500.00|8000.00"
            }
          }
        },
        {
          "tool": "legacy_aggregate_subtotal",
          "label": "wrong_tool_semantic",
          "confidence": 0.4037249982357025,
          "valid_confidence": 0.3830156922340393,
          "margin": 0.020709306001663208,
          "user_request": "Run a Q4 2024 (Oct 1 - Dec 31) compliance audit on Acme Corp (account ACC-12345). Pull the account balance and Q4 transactions, categorize at least one transaction, run a US-region compliance check for the period, calculate the subtotal of all transaction amounts in USD, and submit the audit report with keys: flagged_count (int), total_usd (str), compliance_status (str), txn_count (int).",
          "candidate_call": {
            "name": "legacy_aggregate_subtotal",
            "arguments": {
              "amounts": "5000.00|12500.00|8000.00"
            }
          }
        },
        {
          "tool": "legacy_aggregate_subtotal",
          "label": "wrong_tool_semantic",
          "confidence": 0.4037249982357025,
          "valid_confidence": 0.3830156922340393,
          "margin": 0.020709306001663208,
          "user_request": "Run a Q4 2024 (Oct 1 - Dec 31) compliance audit on Acme Corp (account ACC-12345). Pull the account balance and Q4 transactions, categorize at least one transaction, run a US-region compliance check for the period, calculate the subtotal of all transaction amounts in USD, and submit the audit report with keys: flagged_count (int), total_usd (str), compliance_status (str), txn_count (int).",
          "candidate_call": {
            "name": "legacy_aggregate_subtotal",
            "arguments": {
              "amounts": "5000.00|12500.00|8000.00"
            }
          }
        },
        {
          "tool": "legacy_aggregate_subtotal",
          "label": "wrong_tool_semantic",
          "confidence": 0.4037249982357025,
          "valid_confidence": 0.3830156922340393,
          "margin": 0.020709306001663208,
          "user_request": "Run a Q4 2024 (Oct 1 - Dec 31) compliance audit on Acme Corp (account ACC-12345). Pull the account balance and Q4 transactions, categorize at least one transaction, run a US-region compliance check for the period, calculate the subtotal of all transaction amounts in USD, and submit the audit report with keys: flagged_count (int), total_usd (str), compliance_status (str), txn_count (int).",
          "candidate_call": {
            "name": "legacy_aggregate_subtotal",
            "arguments": {
              "amounts": "5000.00|12500.00|8000.00"
            }
          }
        },
        {
          "tool": "legacy_aggregate_subtotal",
          "label": "wrong_tool_semantic",
          "confidence": 0.4037249982357025,
          "valid_confidence": 0.3830156922340393,
          "margin": 0.020709306001663208,
          "user_request": "Run a Q4 2024 (Oct 1 - Dec 31) compliance audit on Acme Corp (account ACC-12345). Pull the account balance and Q4 transactions, categorize at least one transaction, run a US-region compliance check for the period, calculate the subtotal of all transaction amounts in USD, and submit the audit report with keys: flagged_count (int), total_usd (str), compliance_status (str), txn_count (int).",
          "candidate_call": {
            "name": "legacy_aggregate_subtotal",
            "arguments": {
              "amounts": "5000.00|12500.00|8000.00"
            }
          }
        },
        {
          "tool": "legacy_aggregate_subtotal",
          "label": "wrong_tool_semantic",
          "confidence": 0.4037249982357025,
          "valid_confidence": 0.3830156922340393,
          "margin": 0.020709306001663208,
          "user_request": "Run a Q4 2024 (Oct 1 - Dec 31) compliance audit on Acme Corp (account ACC-12345). Pull the account balance and Q4 transactions, categorize at least one transaction, run a US-region compliance check for the period, calculate the subtotal of all transaction amounts in USD, and submit the audit report with keys: flagged_count (int), total_usd (str), compliance_status (str), txn_count (int).",
          "candidate_call": {
            "name": "legacy_aggregate_subtotal",
            "arguments": {
              "amounts": "5000.00|12500.00|8000.00"
            }
          }
        },
        {
          "tool": "legacy_aggregate_subtotal",
          "label": "wrong_tool_semantic",
          "confidence": 0.4037249982357025,
          "valid_confidence": 0.3830156922340393,
          "margin": 0.020709306001663208,
          "user_request": "Run a Q4 2024 (Oct 1 - Dec 31) compliance audit on Acme Corp (account ACC-12345). Pull the account balance and Q4 transactions, categorize at least one transaction, run a US-region compliance check for the period, calculate the subtotal of all transaction amounts in USD, and submit the audit report with keys: flagged_count (int), total_usd (str), compliance_status (str), txn_count (int).",
          "candidate_call": {
            "name": "legacy_aggregate_subtotal",
            "arguments": {
              "amounts": "5000.00|12500.00|8000.00"
            }
          }
        },
        {
          "tool": "legacy_aggregate_subtotal",
          "label": "wrong_tool_semantic",
          "confidence": 0.4037249982357025,
          "valid_confidence": 0.3830156922340393,
          "margin": 0.020709306001663208,
          "user_request": "Run a Q4 2024 (Oct 1 - Dec 31) compliance audit on Acme Corp (account ACC-12345). Pull the account balance and Q4 transactions, categorize at least one transaction, run a US-region compliance check for the period, calculate the subtotal of all transaction amounts in USD, and submit the audit report with keys: flagged_count (int), total_usd (str), compliance_status (str), txn_count (int).",
          "candidate_call": {
            "name": "legacy_aggregate_subtotal",
            "arguments": {
              "amounts": "5000.00|12500.00|8000.00"
            }
          }
        },
        {
          "tool": "legacy_aggregate_subtotal",
          "label": "wrong_tool_semantic",
          "confidence": 0.4037249982357025,
          "valid_confidence": 0.3830156922340393,
          "margin": 0.020709306001663208,
          "user_request": "Run a Q4 2024 (Oct 1 - Dec 31) compliance audit on Acme Corp (account ACC-12345). Pull the account balance and Q4 transactions, categorize at least one transaction, run a US-region compliance check for the period, calculate the subtotal of all transaction amounts in USD, and submit the audit report with keys: flagged_count (int), total_usd (str), compliance_status (str), txn_count (int).",
          "candidate_call": {
            "name": "legacy_aggregate_subtotal",
            "arguments": {
              "amounts": "5000.00|12500.00|8000.00"
            }
          }
        },
        {
          "tool": "legacy_aggregate_subtotal",
          "label": "wrong_tool_semantic",
          "confidence": 0.4037249982357025,
          "valid_confidence": 0.3830156922340393,
          "margin": 0.020709306001663208,
          "user_request": "Run a Q4 2024 (Oct 1 - Dec 31) compliance audit on Acme Corp (account ACC-12345). Pull the account balance and Q4 transactions, categorize at least one transaction, run a US-region compliance check for the period, calculate the subtotal of all transaction amounts in USD, and submit the audit report with keys: flagged_count (int), total_usd (str), compliance_status (str), txn_count (int).",
          "candidate_call": {
            "name": "legacy_aggregate_subtotal",
            "arguments": {
              "amounts": "5000.00|12500.00|8000.00"
            }
          }
        },
        {
          "tool": "legacy_aggregate_subtotal",
          "label": "wrong_tool_semantic",
          "confidence": 0.4037249982357025,
          "valid_confidence": 0.3830156922340393,
          "margin": 0.020709306001663208,
          "user_request": "Run a Q4 2024 (Oct 1 - Dec 31) compliance audit on Acme Corp (account ACC-12345). Pull the account balance and Q4 transactions, categorize at least one transaction, run a US-region compliance check for the period, calculate the subtotal of all transaction amounts in USD, and submit the audit report with keys: flagged_count (int), total_usd (str), compliance_status (str), txn_count (int).",
          "candidate_call": {
            "name": "legacy_aggregate_subtotal",
            "arguments": {
              "amounts": "5000.00|12500.00|8000.00"
            }
          }
        },
        {
          "tool": "legacy_aggregate_subtotal",
          "label": "wrong_tool_semantic",
          "confidence": 0.4037249982357025,
          "valid_confidence": 0.3830156922340393,
          "margin": 0.020709306001663208,
          "user_request": "Run a Q4 2024 (Oct 1 - Dec 31) compliance audit on Acme Corp (account ACC-12345). Pull the account balance and Q4 transactions, categorize at least one transaction, run a US-region compliance check for the period, calculate the subtotal of all transaction amounts in USD, and submit the audit report with keys: flagged_count (int), total_usd (str), compliance_status (str), txn_count (int).",
          "candidate_call": {
            "name": "legacy_aggregate_subtotal",
            "arguments": {
              "amounts": "5000.00|12500.00|8000.00"
            }
          }
        },
        {
          "tool": "legacy_aggregate_subtotal",
          "label": "wrong_tool_semantic",
          "confidence": 0.4037249982357025,
          "valid_confidence": 0.3830156922340393,
          "margin": 0.020709306001663208,
          "user_request": "Run a Q4 2024 (Oct 1 - Dec 31) compliance audit on Acme Corp (account ACC-12345). Pull the account balance and Q4 transactions, categorize at least one transaction, run a US-region compliance check for the period, calculate the subtotal of all transaction amounts in USD, and submit the audit report with keys: flagged_count (int), total_usd (str), compliance_status (str), txn_count (int).",
          "candidate_call": {
            "name": "legacy_aggregate_subtotal",
            "arguments": {
              "amounts": "5000.00|12500.00|8000.00"
            }
          }
        },
        {
          "tool": "legacy_aggregate_subtotal",
          "label": "wrong_tool_semantic",
          "confidence": 0.4037249982357025,
          "valid_confidence": 0.3830156922340393,
          "margin": 0.020709306001663208,
          "user_request": "Run a Q4 2024 (Oct 1 - Dec 31) compliance audit on Acme Corp (account ACC-12345). Pull the account balance and Q4 transactions, categorize at least one transaction, run a US-region compliance check for the period, calculate the subtotal of all transaction amounts in USD, and submit the audit report with keys: flagged_count (int), total_usd (str), compliance_status (str), txn_count (int).",
          "candidate_call": {
            "name": "legacy_aggregate_subtotal",
            "arguments": {
              "amounts": "5000.00|12500.00|8000.00"
            }
          }
        },
        {
          "tool": "legacy_aggregate_subtotal",
          "label": "wrong_tool_semantic",
          "confidence": 0.4037249982357025,
          "valid_confidence": 0.3830156922340393,
          "margin": 0.020709306001663208,
          "user_request": "Run a Q4 2024 (Oct 1 - Dec 31) compliance audit on Acme Corp (account ACC-12345). Pull the account balance and Q4 transactions, categorize at least one transaction, run a US-region compliance check for the period, calculate the subtotal of all transaction amounts in USD, and submit the audit report with keys: flagged_count (int), total_usd (str), compliance_status (str), txn_count (int).",
          "candidate_call": {
            "name": "legacy_aggregate_subtotal",
            "arguments": {
              "amounts": "5000.00|12500.00|8000.00"
            }
          }
        },
        {
          "tool": "legacy_aggregate_subtotal",
          "label": "wrong_tool_semantic",
          "confidence": 0.4037249982357025,
          "valid_confidence": 0.3830156922340393,
          "margin": 0.020709306001663208,
          "user_request": "Run a Q4 2024 (Oct 1 - Dec 31) compliance audit on Acme Corp (account ACC-12345). Pull the account balance and Q4 transactions, categorize at least one transaction, run a US-region compliance check for the period, calculate the subtotal of all transaction amounts in USD, and submit the audit report with keys: flagged_count (int), total_usd (str), compliance_status (str), txn_count (int).",
          "candidate_call": {
            "name": "legacy_aggregate_subtotal",
            "arguments": {
              "amounts": "5000.00|12500.00|8000.00"
            }
          }
        },
        {
          "tool": "legacy_aggregate_subtotal",
          "label": "wrong_tool_semantic",
          "confidence": 0.4037249982357025,
          "valid_confidence": 0.3830156922340393,
          "margin": 0.020709306001663208,
          "user_request": "Run a Q4 2024 (Oct 1 - Dec 31) compliance audit on Acme Corp (account ACC-12345). Pull the account balance and Q4 transactions, categorize at least one transaction, run a US-region compliance check for the period, calculate the subtotal of all transaction amounts in USD, and submit the audit report with keys: flagged_count (int), total_usd (str), compliance_status (str), txn_count (int).",
          "candidate_call": {
            "name": "legacy_aggregate_subtotal",
            "arguments": {
              "amounts": "5000.00|12500.00|8000.00"
            }
          }
        },
        {
          "tool": "legacy_aggregate_subtotal",
          "label": "wrong_tool_semantic",
          "confidence": 0.4037249982357025,
          "valid_confidence": 0.3830156922340393,
          "margin": 0.020709306001663208,
          "user_request": "Run a Q4 2024 (Oct 1 - Dec 31) compliance audit on Acme Corp (account ACC-12345). Pull the account balance and Q4 transactions, categorize at least one transaction, run a US-region compliance check for the period, calculate the subtotal of all transaction amounts in USD, and submit the audit report with keys: flagged_count (int), total_usd (str), compliance_status (str), txn_count (int).",
          "candidate_call": {
            "name": "legacy_aggregate_subtotal",
            "arguments": {
              "amounts": "5000.00|12500.00|8000.00"
            }
          }
        },
        {
          "tool": "legacy_aggregate_subtotal",
          "label": "wrong_tool_semantic",
          "confidence": 0.4037249982357025,
          "valid_confidence": 0.3830156922340393,
          "margin": 0.020709306001663208,
          "user_request": "Run a Q4 2024 (Oct 1 - Dec 31) compliance audit on Acme Corp (account ACC-12345). Pull the account balance and Q4 transactions, categorize at least one transaction, run a US-region compliance check for the period, calculate the subtotal of all transaction amounts in USD, and submit the audit report with keys: flagged_count (int), total_usd (str), compliance_status (str), txn_count (int).",
          "candidate_call": {
            "name": "legacy_aggregate_subtotal",
            "arguments": {
              "amounts": "5000.00|12500.00|8000.00"
            }
          }
        },
        {
          "tool": "legacy_aggregate_subtotal",
          "label": "wrong_tool_semantic",
          "confidence": 0.4037249982357025,
          "valid_confidence": 0.3830156922340393,
          "margin": 0.020709306001663208,
          "user_request": "Run a Q4 2024 (Oct 1 - Dec 31) compliance audit on Acme Corp (account ACC-12345). Pull the account balance and Q4 transactions, categorize at least one transaction, run a US-region compliance check for the period, calculate the subtotal of all transaction amounts in USD, and submit the audit report with keys: flagged_count (int), total_usd (str), compliance_status (str), txn_count (int).",
          "candidate_call": {
            "name": "legacy_aggregate_subtotal",
            "arguments": {
              "amounts": "5000.00|12500.00|8000.00"
            }
          }
        },
        {
          "tool": "get_records",
          "label": "wrong_tool_semantic",
          "confidence": 0.3361436128616333,
          "valid_confidence": 0.31428298354148865,
          "margin": 0.021860629320144653,
          "user_request": "Check drug interactions for patient John Doe's current medication.",
          "candidate_call": {
            "name": "get_records",
            "arguments": {
              "patient_id": "PT-7829"
            }
          }
        },
        {
          "tool": "get_records",
          "label": "wrong_tool_semantic",
          "confidence": 0.3361436128616333,
          "valid_confidence": 0.31428298354148865,
          "margin": 0.021860629320144653,
          "user_request": "Check drug interactions for patient John Doe's current medication.",
          "candidate_call": {
            "name": "get_records",
            "arguments": {
              "patient_id": "PT-7829"
            }
          }
        },
        {
          "tool": "get_records",
          "label": "wrong_tool_semantic",
          "confidence": 0.3361436128616333,
          "valid_confidence": 0.31428298354148865,
          "margin": 0.021860629320144653,
          "user_request": "Check drug interactions for patient John Doe's current medication.",
          "candidate_call": {
            "name": "get_records",
            "arguments": {
              "patient_id": "PT-7829"
            }
          }
        },
        {
          "tool": "get_records",
          "label": "wrong_tool_semantic",
          "confidence": 0.3361436128616333,
          "valid_confidence": 0.31428298354148865,
          "margin": 0.021860629320144653,
          "user_request": "Check drug interactions for patient John Doe's current medication.",
          "candidate_call": {
            "name": "get_records",
            "arguments": {
              "patient_id": "PT-7829"
            }
          }
        }
      ]
    },
    {
      "path": "/content/rust_smoke.jsonl",
      "kind": "rust_smoke",
      "rows": 30,
      "classifier_scores": 69,
      "predicted_labels": {
        "wrong_tool_semantic": 66,
        "valid": 2,
        "wrong_arguments_semantic": 1
      },
      "successful_accurate_scores_as_valid_proxy": 62,
      "valid_recall_proxy": 0.016129032258064516,
      "valid_proxy_predicted_labels": {
        "wrong_tool_semantic": 60,
        "wrong_arguments_semantic": 1,
        "valid": 1
      },
      "false_objection_tools": {
        "summarize": 16,
        "fetch_sales_data": 10,
        "analyze_sales": 10,
        "get_country_info": 9,
        "report": 9,
        "fetch": 7
      },
      "false_objection_scenarios": {
        "sequential_3step": 29,
        "basic_2step": 18,
        "error_recovery": 14
      },
      "terminal_false_objection_tools": {
        "summarize": 16,
        "report": 9
      },
      "numeric_string_false_objection_tools": {
        "fetch": 7,
        "summarize": 7
      },
      "fixed_width_numeric_false_objection_tools": {
        "fetch": 7,
        "summarize": 7
      }
    }
  ]
}