| { |
| "schema_version": "toolcall-verifier-telemetry-diagnostics/v1", |
| "telemetry_only": true, |
| "used_for_training": false, |
| "notes": [ |
| "Use these rows to diagnose false objections and dataset gaps.", |
| "Do not treat top-k telemetry from the current artifact as promotion evidence." |
| ], |
| "files": [ |
| { |
| "path": "/content/proxy_classifier_budget_8192.jsonl", |
| "kind": "proxy_classifier", |
| "rows": 1564, |
| "predicted_labels": { |
| "wrong_tool_semantic": 1292, |
| "valid": 188, |
| "wrong_arguments_semantic": 84 |
| }, |
| "actions": { |
| "shadow_only": 1376, |
| "allow": 188 |
| }, |
| "terminal_or_terminal_like": { |
| "rows": 405, |
| "predicted_labels": { |
| "wrong_tool_semantic": 343, |
| "valid": 38, |
| "wrong_arguments_semantic": 24 |
| }, |
| "tools": { |
| "get_skill_summary": 100, |
| "summarize": 59, |
| "respond": 54, |
| "submit_report": 40, |
| "report": 30, |
| "submit_hiring_decision": 22, |
| "present": 20, |
| "recommend": 20, |
| "diagnose": 20, |
| "submit_audit_report": 20, |
| "legacy_submit_audit": 20 |
| } |
| }, |
| "numeric_string_candidates": { |
| "rows": 51, |
| "predicted_labels": { |
| "wrong_tool_semantic": 50, |
| "valid": 1 |
| }, |
| "tools": { |
| "fetch": 50, |
| "submit_audit_report": 1 |
| } |
| }, |
| "fixed_width_numeric_candidates": { |
| "rows": 30, |
| "predicted_labels": { |
| "wrong_tool_semantic": 30 |
| }, |
| "tools": { |
| "fetch": 30 |
| } |
| }, |
| "non_valid_top_label_with_close_valid_margin": [ |
| { |
| "tool": "submit_audit_report", |
| "label": "wrong_arguments_semantic", |
| "confidence": 0.34632304310798645, |
| "valid_confidence": 0.33387550711631775, |
| "margin": 0.012447535991668701, |
| "user_request": "Run our Q4 2024 expense audit. Flag any transaction of $5,000 or more from vendors NOT on our approved list. Submit the audit report with: comma-separated transaction IDs, total flagged amount in USD, and the vendor of the single largest flagged transaction.", |
| "candidate_call": { |
| "name": "submit_audit_report", |
| "arguments": { |
| "transaction_ids": "TX-1001,TX-1002,TX-1003,TX-1005,TX-1006,TX-1007,TX-1008,TX-1012", |
| "total_flagged_usd": "73160.00", |
| "top_vendor": "Wonka Industries" |
| } |
| } |
| }, |
| { |
| "tool": "legacy_aggregate_subtotal", |
| "label": "wrong_tool_semantic", |
| "confidence": 0.4037249982357025, |
| "valid_confidence": 0.3830156922340393, |
| "margin": 0.020709306001663208, |
| "user_request": "Run a Q4 2024 (Oct 1 - Dec 31) compliance audit on Acme Corp (account ACC-12345). Pull the account balance and Q4 transactions, categorize at least one transaction, run a US-region compliance check for the period, calculate the subtotal of all transaction amounts in USD, and submit the audit report with keys: flagged_count (int), total_usd (str), compliance_status (str), txn_count (int).", |
| "candidate_call": { |
| "name": "legacy_aggregate_subtotal", |
| "arguments": { |
| "amounts": "5000.00|12500.00|8000.00" |
| } |
| } |
| }, |
| { |
| "tool": "legacy_aggregate_subtotal", |
| "label": "wrong_tool_semantic", |
| "confidence": 0.4037249982357025, |
| "valid_confidence": 0.3830156922340393, |
| "margin": 0.020709306001663208, |
| "user_request": "Run a Q4 2024 (Oct 1 - Dec 31) compliance audit on Acme Corp (account ACC-12345). Pull the account balance and Q4 transactions, categorize at least one transaction, run a US-region compliance check for the period, calculate the subtotal of all transaction amounts in USD, and submit the audit report with keys: flagged_count (int), total_usd (str), compliance_status (str), txn_count (int).", |
| "candidate_call": { |
| "name": "legacy_aggregate_subtotal", |
| "arguments": { |
| "amounts": "5000.00|12500.00|8000.00" |
| } |
| } |
| }, |
| { |
| "tool": "legacy_aggregate_subtotal", |
| "label": "wrong_tool_semantic", |
| "confidence": 0.4037249982357025, |
| "valid_confidence": 0.3830156922340393, |
| "margin": 0.020709306001663208, |
| "user_request": "Run a Q4 2024 (Oct 1 - Dec 31) compliance audit on Acme Corp (account ACC-12345). Pull the account balance and Q4 transactions, categorize at least one transaction, run a US-region compliance check for the period, calculate the subtotal of all transaction amounts in USD, and submit the audit report with keys: flagged_count (int), total_usd (str), compliance_status (str), txn_count (int).", |
| "candidate_call": { |
| "name": "legacy_aggregate_subtotal", |
| "arguments": { |
| "amounts": "5000.00|12500.00|8000.00" |
| } |
| } |
| }, |
| { |
| "tool": "legacy_aggregate_subtotal", |
| "label": "wrong_tool_semantic", |
| "confidence": 0.4037249982357025, |
| "valid_confidence": 0.3830156922340393, |
| "margin": 0.020709306001663208, |
| "user_request": "Run a Q4 2024 (Oct 1 - Dec 31) compliance audit on Acme Corp (account ACC-12345). Pull the account balance and Q4 transactions, categorize at least one transaction, run a US-region compliance check for the period, calculate the subtotal of all transaction amounts in USD, and submit the audit report with keys: flagged_count (int), total_usd (str), compliance_status (str), txn_count (int).", |
| "candidate_call": { |
| "name": "legacy_aggregate_subtotal", |
| "arguments": { |
| "amounts": "5000.00|12500.00|8000.00" |
| } |
| } |
| }, |
| { |
| "tool": "legacy_aggregate_subtotal", |
| "label": "wrong_tool_semantic", |
| "confidence": 0.4037249982357025, |
| "valid_confidence": 0.3830156922340393, |
| "margin": 0.020709306001663208, |
| "user_request": "Run a Q4 2024 (Oct 1 - Dec 31) compliance audit on Acme Corp (account ACC-12345). Pull the account balance and Q4 transactions, categorize at least one transaction, run a US-region compliance check for the period, calculate the subtotal of all transaction amounts in USD, and submit the audit report with keys: flagged_count (int), total_usd (str), compliance_status (str), txn_count (int).", |
| "candidate_call": { |
| "name": "legacy_aggregate_subtotal", |
| "arguments": { |
| "amounts": "5000.00|12500.00|8000.00" |
| } |
| } |
| }, |
| { |
| "tool": "legacy_aggregate_subtotal", |
| "label": "wrong_tool_semantic", |
| "confidence": 0.4037249982357025, |
| "valid_confidence": 0.3830156922340393, |
| "margin": 0.020709306001663208, |
| "user_request": "Run a Q4 2024 (Oct 1 - Dec 31) compliance audit on Acme Corp (account ACC-12345). Pull the account balance and Q4 transactions, categorize at least one transaction, run a US-region compliance check for the period, calculate the subtotal of all transaction amounts in USD, and submit the audit report with keys: flagged_count (int), total_usd (str), compliance_status (str), txn_count (int).", |
| "candidate_call": { |
| "name": "legacy_aggregate_subtotal", |
| "arguments": { |
| "amounts": "5000.00|12500.00|8000.00" |
| } |
| } |
| }, |
| { |
| "tool": "legacy_aggregate_subtotal", |
| "label": "wrong_tool_semantic", |
| "confidence": 0.4037249982357025, |
| "valid_confidence": 0.3830156922340393, |
| "margin": 0.020709306001663208, |
| "user_request": "Run a Q4 2024 (Oct 1 - Dec 31) compliance audit on Acme Corp (account ACC-12345). Pull the account balance and Q4 transactions, categorize at least one transaction, run a US-region compliance check for the period, calculate the subtotal of all transaction amounts in USD, and submit the audit report with keys: flagged_count (int), total_usd (str), compliance_status (str), txn_count (int).", |
| "candidate_call": { |
| "name": "legacy_aggregate_subtotal", |
| "arguments": { |
| "amounts": "5000.00|12500.00|8000.00" |
| } |
| } |
| }, |
| { |
| "tool": "legacy_aggregate_subtotal", |
| "label": "wrong_tool_semantic", |
| "confidence": 0.4037249982357025, |
| "valid_confidence": 0.3830156922340393, |
| "margin": 0.020709306001663208, |
| "user_request": "Run a Q4 2024 (Oct 1 - Dec 31) compliance audit on Acme Corp (account ACC-12345). Pull the account balance and Q4 transactions, categorize at least one transaction, run a US-region compliance check for the period, calculate the subtotal of all transaction amounts in USD, and submit the audit report with keys: flagged_count (int), total_usd (str), compliance_status (str), txn_count (int).", |
| "candidate_call": { |
| "name": "legacy_aggregate_subtotal", |
| "arguments": { |
| "amounts": "5000.00|12500.00|8000.00" |
| } |
| } |
| }, |
| { |
| "tool": "legacy_aggregate_subtotal", |
| "label": "wrong_tool_semantic", |
| "confidence": 0.4037249982357025, |
| "valid_confidence": 0.3830156922340393, |
| "margin": 0.020709306001663208, |
| "user_request": "Run a Q4 2024 (Oct 1 - Dec 31) compliance audit on Acme Corp (account ACC-12345). Pull the account balance and Q4 transactions, categorize at least one transaction, run a US-region compliance check for the period, calculate the subtotal of all transaction amounts in USD, and submit the audit report with keys: flagged_count (int), total_usd (str), compliance_status (str), txn_count (int).", |
| "candidate_call": { |
| "name": "legacy_aggregate_subtotal", |
| "arguments": { |
| "amounts": "5000.00|12500.00|8000.00" |
| } |
| } |
| }, |
| { |
| "tool": "legacy_aggregate_subtotal", |
| "label": "wrong_tool_semantic", |
| "confidence": 0.4037249982357025, |
| "valid_confidence": 0.3830156922340393, |
| "margin": 0.020709306001663208, |
| "user_request": "Run a Q4 2024 (Oct 1 - Dec 31) compliance audit on Acme Corp (account ACC-12345). Pull the account balance and Q4 transactions, categorize at least one transaction, run a US-region compliance check for the period, calculate the subtotal of all transaction amounts in USD, and submit the audit report with keys: flagged_count (int), total_usd (str), compliance_status (str), txn_count (int).", |
| "candidate_call": { |
| "name": "legacy_aggregate_subtotal", |
| "arguments": { |
| "amounts": "5000.00|12500.00|8000.00" |
| } |
| } |
| }, |
| { |
| "tool": "legacy_aggregate_subtotal", |
| "label": "wrong_tool_semantic", |
| "confidence": 0.4037249982357025, |
| "valid_confidence": 0.3830156922340393, |
| "margin": 0.020709306001663208, |
| "user_request": "Run a Q4 2024 (Oct 1 - Dec 31) compliance audit on Acme Corp (account ACC-12345). Pull the account balance and Q4 transactions, categorize at least one transaction, run a US-region compliance check for the period, calculate the subtotal of all transaction amounts in USD, and submit the audit report with keys: flagged_count (int), total_usd (str), compliance_status (str), txn_count (int).", |
| "candidate_call": { |
| "name": "legacy_aggregate_subtotal", |
| "arguments": { |
| "amounts": "5000.00|12500.00|8000.00" |
| } |
| } |
| }, |
| { |
| "tool": "legacy_aggregate_subtotal", |
| "label": "wrong_tool_semantic", |
| "confidence": 0.4037249982357025, |
| "valid_confidence": 0.3830156922340393, |
| "margin": 0.020709306001663208, |
| "user_request": "Run a Q4 2024 (Oct 1 - Dec 31) compliance audit on Acme Corp (account ACC-12345). Pull the account balance and Q4 transactions, categorize at least one transaction, run a US-region compliance check for the period, calculate the subtotal of all transaction amounts in USD, and submit the audit report with keys: flagged_count (int), total_usd (str), compliance_status (str), txn_count (int).", |
| "candidate_call": { |
| "name": "legacy_aggregate_subtotal", |
| "arguments": { |
| "amounts": "5000.00|12500.00|8000.00" |
| } |
| } |
| }, |
| { |
| "tool": "legacy_aggregate_subtotal", |
| "label": "wrong_tool_semantic", |
| "confidence": 0.4037249982357025, |
| "valid_confidence": 0.3830156922340393, |
| "margin": 0.020709306001663208, |
| "user_request": "Run a Q4 2024 (Oct 1 - Dec 31) compliance audit on Acme Corp (account ACC-12345). Pull the account balance and Q4 transactions, categorize at least one transaction, run a US-region compliance check for the period, calculate the subtotal of all transaction amounts in USD, and submit the audit report with keys: flagged_count (int), total_usd (str), compliance_status (str), txn_count (int).", |
| "candidate_call": { |
| "name": "legacy_aggregate_subtotal", |
| "arguments": { |
| "amounts": "5000.00|12500.00|8000.00" |
| } |
| } |
| }, |
| { |
| "tool": "legacy_aggregate_subtotal", |
| "label": "wrong_tool_semantic", |
| "confidence": 0.4037249982357025, |
| "valid_confidence": 0.3830156922340393, |
| "margin": 0.020709306001663208, |
| "user_request": "Run a Q4 2024 (Oct 1 - Dec 31) compliance audit on Acme Corp (account ACC-12345). Pull the account balance and Q4 transactions, categorize at least one transaction, run a US-region compliance check for the period, calculate the subtotal of all transaction amounts in USD, and submit the audit report with keys: flagged_count (int), total_usd (str), compliance_status (str), txn_count (int).", |
| "candidate_call": { |
| "name": "legacy_aggregate_subtotal", |
| "arguments": { |
| "amounts": "5000.00|12500.00|8000.00" |
| } |
| } |
| }, |
| { |
| "tool": "legacy_aggregate_subtotal", |
| "label": "wrong_tool_semantic", |
| "confidence": 0.4037249982357025, |
| "valid_confidence": 0.3830156922340393, |
| "margin": 0.020709306001663208, |
| "user_request": "Run a Q4 2024 (Oct 1 - Dec 31) compliance audit on Acme Corp (account ACC-12345). Pull the account balance and Q4 transactions, categorize at least one transaction, run a US-region compliance check for the period, calculate the subtotal of all transaction amounts in USD, and submit the audit report with keys: flagged_count (int), total_usd (str), compliance_status (str), txn_count (int).", |
| "candidate_call": { |
| "name": "legacy_aggregate_subtotal", |
| "arguments": { |
| "amounts": "5000.00|12500.00|8000.00" |
| } |
| } |
| }, |
| { |
| "tool": "legacy_aggregate_subtotal", |
| "label": "wrong_tool_semantic", |
| "confidence": 0.4037249982357025, |
| "valid_confidence": 0.3830156922340393, |
| "margin": 0.020709306001663208, |
| "user_request": "Run a Q4 2024 (Oct 1 - Dec 31) compliance audit on Acme Corp (account ACC-12345). Pull the account balance and Q4 transactions, categorize at least one transaction, run a US-region compliance check for the period, calculate the subtotal of all transaction amounts in USD, and submit the audit report with keys: flagged_count (int), total_usd (str), compliance_status (str), txn_count (int).", |
| "candidate_call": { |
| "name": "legacy_aggregate_subtotal", |
| "arguments": { |
| "amounts": "5000.00|12500.00|8000.00" |
| } |
| } |
| }, |
| { |
| "tool": "legacy_aggregate_subtotal", |
| "label": "wrong_tool_semantic", |
| "confidence": 0.4037249982357025, |
| "valid_confidence": 0.3830156922340393, |
| "margin": 0.020709306001663208, |
| "user_request": "Run a Q4 2024 (Oct 1 - Dec 31) compliance audit on Acme Corp (account ACC-12345). Pull the account balance and Q4 transactions, categorize at least one transaction, run a US-region compliance check for the period, calculate the subtotal of all transaction amounts in USD, and submit the audit report with keys: flagged_count (int), total_usd (str), compliance_status (str), txn_count (int).", |
| "candidate_call": { |
| "name": "legacy_aggregate_subtotal", |
| "arguments": { |
| "amounts": "5000.00|12500.00|8000.00" |
| } |
| } |
| }, |
| { |
| "tool": "legacy_aggregate_subtotal", |
| "label": "wrong_tool_semantic", |
| "confidence": 0.4037249982357025, |
| "valid_confidence": 0.3830156922340393, |
| "margin": 0.020709306001663208, |
| "user_request": "Run a Q4 2024 (Oct 1 - Dec 31) compliance audit on Acme Corp (account ACC-12345). Pull the account balance and Q4 transactions, categorize at least one transaction, run a US-region compliance check for the period, calculate the subtotal of all transaction amounts in USD, and submit the audit report with keys: flagged_count (int), total_usd (str), compliance_status (str), txn_count (int).", |
| "candidate_call": { |
| "name": "legacy_aggregate_subtotal", |
| "arguments": { |
| "amounts": "5000.00|12500.00|8000.00" |
| } |
| } |
| }, |
| { |
| "tool": "legacy_aggregate_subtotal", |
| "label": "wrong_tool_semantic", |
| "confidence": 0.4037249982357025, |
| "valid_confidence": 0.3830156922340393, |
| "margin": 0.020709306001663208, |
| "user_request": "Run a Q4 2024 (Oct 1 - Dec 31) compliance audit on Acme Corp (account ACC-12345). Pull the account balance and Q4 transactions, categorize at least one transaction, run a US-region compliance check for the period, calculate the subtotal of all transaction amounts in USD, and submit the audit report with keys: flagged_count (int), total_usd (str), compliance_status (str), txn_count (int).", |
| "candidate_call": { |
| "name": "legacy_aggregate_subtotal", |
| "arguments": { |
| "amounts": "5000.00|12500.00|8000.00" |
| } |
| } |
| }, |
| { |
| "tool": "legacy_aggregate_subtotal", |
| "label": "wrong_tool_semantic", |
| "confidence": 0.4037249982357025, |
| "valid_confidence": 0.3830156922340393, |
| "margin": 0.020709306001663208, |
| "user_request": "Run a Q4 2024 (Oct 1 - Dec 31) compliance audit on Acme Corp (account ACC-12345). Pull the account balance and Q4 transactions, categorize at least one transaction, run a US-region compliance check for the period, calculate the subtotal of all transaction amounts in USD, and submit the audit report with keys: flagged_count (int), total_usd (str), compliance_status (str), txn_count (int).", |
| "candidate_call": { |
| "name": "legacy_aggregate_subtotal", |
| "arguments": { |
| "amounts": "5000.00|12500.00|8000.00" |
| } |
| } |
| }, |
| { |
| "tool": "get_records", |
| "label": "wrong_tool_semantic", |
| "confidence": 0.3361436128616333, |
| "valid_confidence": 0.31428298354148865, |
| "margin": 0.021860629320144653, |
| "user_request": "Check drug interactions for patient John Doe's current medication.", |
| "candidate_call": { |
| "name": "get_records", |
| "arguments": { |
| "patient_id": "PT-7829" |
| } |
| } |
| }, |
| { |
| "tool": "get_records", |
| "label": "wrong_tool_semantic", |
| "confidence": 0.3361436128616333, |
| "valid_confidence": 0.31428298354148865, |
| "margin": 0.021860629320144653, |
| "user_request": "Check drug interactions for patient John Doe's current medication.", |
| "candidate_call": { |
| "name": "get_records", |
| "arguments": { |
| "patient_id": "PT-7829" |
| } |
| } |
| }, |
| { |
| "tool": "get_records", |
| "label": "wrong_tool_semantic", |
| "confidence": 0.3361436128616333, |
| "valid_confidence": 0.31428298354148865, |
| "margin": 0.021860629320144653, |
| "user_request": "Check drug interactions for patient John Doe's current medication.", |
| "candidate_call": { |
| "name": "get_records", |
| "arguments": { |
| "patient_id": "PT-7829" |
| } |
| } |
| }, |
| { |
| "tool": "get_records", |
| "label": "wrong_tool_semantic", |
| "confidence": 0.3361436128616333, |
| "valid_confidence": 0.31428298354148865, |
| "margin": 0.021860629320144653, |
| "user_request": "Check drug interactions for patient John Doe's current medication.", |
| "candidate_call": { |
| "name": "get_records", |
| "arguments": { |
| "patient_id": "PT-7829" |
| } |
| } |
| } |
| ] |
| }, |
| { |
| "path": "/content/rust_smoke.jsonl", |
| "kind": "rust_smoke", |
| "rows": 30, |
| "classifier_scores": 69, |
| "predicted_labels": { |
| "wrong_tool_semantic": 66, |
| "valid": 2, |
| "wrong_arguments_semantic": 1 |
| }, |
| "successful_accurate_scores_as_valid_proxy": 62, |
| "valid_recall_proxy": 0.016129032258064516, |
| "valid_proxy_predicted_labels": { |
| "wrong_tool_semantic": 60, |
| "wrong_arguments_semantic": 1, |
| "valid": 1 |
| }, |
| "false_objection_tools": { |
| "summarize": 16, |
| "fetch_sales_data": 10, |
| "analyze_sales": 10, |
| "get_country_info": 9, |
| "report": 9, |
| "fetch": 7 |
| }, |
| "false_objection_scenarios": { |
| "sequential_3step": 29, |
| "basic_2step": 18, |
| "error_recovery": 14 |
| }, |
| "terminal_false_objection_tools": { |
| "summarize": 16, |
| "report": 9 |
| }, |
| "numeric_string_false_objection_tools": { |
| "fetch": 7, |
| "summarize": 7 |
| }, |
| "fixed_width_numeric_false_objection_tools": { |
| "fetch": 7, |
| "summarize": 7 |
| } |
| } |
| ] |
| } |
