diff --git "a/eval-report.json" "b/eval-report.json" new file mode 100644--- /dev/null +++ "b/eval-report.json" @@ -0,0 +1,5475 @@ +{ + "adapter": "Taimwe/securecoder-30b-pro", + "base": "unsloth/Qwen3-Coder-30B-A3B-Instruct", + "when": "2026-10-04T08:55:35Z", + "sections": [ + { + "section": "tool_calls", + "n_prompts": 80, + "parse_rate": 1.0, + "name_rate": 1.0, + "schema_rate": 1.0 + }, + { + "section": "code_sanity", + "n_prompts": 15, + "n_scored": 15, + "n_truncated": 0, + "truncated_rate": 0.0, + "ast_rate": 0.0, + "compile_rate": 0.0 + }, + { + "section": "security_mcq", + "n_questions": 0, + "accuracy": 0.0 + } + ], + "raw": [ + { + "section": "tool_calls", + "n_prompts": 80, + "parse_rate": 1.0, + "name_rate": 1.0, + "schema_rate": 1.0, + "details": [ + { + "prompt": "I've recently installed a new security system at my home, and I want to ensure e", + "reply_first_160": "\n\n\nfront_door\n\n\n1080p\n\n\n", + "expected_call": true, + "n_calls": 3, + "calls": [ + { + "name": "get_camera_live_feed", + "arguments": { + "camera_id": "front_door", + "stream_quality": "1080p" + } + }, + { + "name": "record_camera_feed", + "arguments": { + "camera_id": "front_door", + "duration": "30" + } + }, + { + "name": "get_recorded_feed", + "arguments": { + "camera_id": "front_garden", + "start_time": "2023-04-22T15:00:00Z", + "end_time": "2023-04-22T17:00:00Z" + } + } + ], + "scores": [ + { + "parse": true, + "name_ok": true, + "schema_ok": true, + "expected_keys": [ + "camera_id", + "stream_quality" + ], + "given_keys": [ + "camera_id", + "stream_quality" + ] + }, + { + "parse": true, + "name_ok": true, + "schema_ok": true, + "expected_keys": [ + "camera_id", + "duration" + ], + "given_keys": [ + "camera_id", + "duration" + ] + }, + { + "parse": true, + "name_ok": true, + "schema_ok": true, + "expected_keys": [ + "camera_id", + "end_time", + "start_time" + ], + "given_keys": [ + "camera_id", + "end_time", + "start_time" + ] + } + ] + }, + { + "prompt": "I've recently equipped my home with various smart devices to enhance my living e", + "reply_first_160": "\n\n\n[\"Philips Hue smart bulbs\", \"Nest Thermostat\", \"Arlo Pro 3 security camera\"]\n\n\n\nthermo123\n\n\n[{\"day\": \"Monday\", \"temperature\": 68, \"time", + "expected_call": true, + "n_calls": 1, + "calls": [ + { + "name": "set_thermostat_schedule", + "arguments": { + "thermostat_id": "thermo123", + "schedule": "[{\"day\": \"Monday\", \"temperature\": 68, \"time\": \"07:00\"}, {\"day\": \"Monday\", \"temperature\": 62, \"time\": \"23:00\"}, {\"day\": \"Tuesday\", \"temperature\": 68, \"time\": \"07:00\"}, {\"day\": \"Tuesday\", \"temperature\": 62, \"time\": \"23:00\"}, {\"day\": \"Wednesday\", \"temperature\": 68, \"time\": \"07:00\"}, {\"day\": \"Wednesday\", \"temperature\": 62, \"time\": \"23:00\"}, {\"day\": \"Thursday\", \"temperature\": 68, \"time\": \"07:00\"}, {\"day\": \"Thursday\", \"temperature\": 62, \"time\": \"23:00\"}, {\"day\": \"Friday\", \"temperature\": 68, \"time\": \"07:00\"}, {\"day\": \"Friday\", \"temperature\": 62, \"time\": \"23:00\"}, {\"day\": \"Saturday\", \"temperature\": 68, \"time\": \"08:00\"}, {\"day\": \"Saturday\", \"temperature\": 62, \"time\": \"23:00\"}, {\"day\": \"Sunday\", \"temperature\": 68, \"time\": \"08:00\"}, {\"day\": \"Sunday\", \"temperature\": 62, \"time\": \"23:00\"}]" + } + } + ], + "scores": [ + { + "parse": true, + "name_ok": true, + "schema_ok": true, + "expected_keys": [ + "schedule", + "thermostat_id" + ], + "given_keys": [ + "schedule", + "thermostat_id" + ] + } + ] + }, + { + "prompt": "I just arrived home and I'm carrying several bags of groceries, so I can't manua", + "reply_first_160": "\n\n\nActivate kitchen lights\n\n\nlight\n\n\n\n\nliving room\n\n\ndim\n\n\nwarm\n\n\n\nSGDO12345678\n\n\na1b2c3d4e5f6g7h8\n\n\n\n\nzone_01\n\n\n30\n\n\n\n\nliving-room-light-001\n\n\nturn_on\n\n\n\n\n\nTH12345\n\n\n68\n\n\n19:0", + "expected_call": true, + "n_calls": 3, + "calls": [ + { + "name": "set_thermostat_schedule", + "arguments": { + "device_id": "TH12345", + "temperature": "68", + "schedule_time": "19:00" + } + }, + { + "name": "set_door_lock_schedule", + "arguments": { + "device_id": "DL54321", + "lock_state": "locked", + "schedule_time": "23:00" + } + }, + { + "name": "set_sprinkler_schedule", + "arguments": { + "device_id": "SP98765", + "interval_days": "2", + "start_time": "06:00" + } + } + ], + "scores": [ + { + "parse": true, + "name_ok": true, + "schema_ok": true, + "expected_keys": [ + "device_id", + "schedule_time", + "temperature" + ], + "given_keys": [ + "device_id", + "schedule_time", + "temperature" + ] + }, + { + "parse": true, + "name_ok": true, + "schema_ok": true, + "expected_keys": [ + "device_id", + "lock_state", + "schedule_time" + ], + "given_keys": [ + "device_id", + "lock_state", + "schedule_time" + ] + }, + { + "parse": true, + "name_ok": true, + "schema_ok": true, + "expected_keys": [ + "device_id", + "interval_days", + "start_time" + ], + "given_keys": [ + "device_id", + "interval_days", + "start_time" + ] + } + ] + }, + { + "prompt": "I would like to adjust the smart window blinds in my living room to precisely 75", + "reply_first_160": "\n\n\nliving room\n\n\n75\n\n\n", + "expected_call": true, + "n_calls": 1, + "calls": [ + { + "name": "set_blind_openness", + "arguments": { + "room": "living room", + "openness_percentage": "75" + } + } + ], + "scores": [ + { + "parse": true, + "name_ok": true, + "schema_ok": true, + "expected_keys": [ + "openness_percentage", + "room" + ], + "given_keys": [ + "openness_percentage", + "room" + ] + } + ] + }, + { + "prompt": "I recently installed a smart fridge model SF-001, and I'm looking to optimize it", + "reply_first_160": "\n\n\nSF-001\n\n\nenergy_saving\n\n\n\n\nTrue\n\n\n", + "expected_call": true, + "n_calls": 1, + "calls": [ + { + "name": "lock_all_smart_doors", + "arguments": { + "confirmation_required": "True" + } + } + ], + "scores": [ + { + "parse": true, + "name_ok": true, + "schema_ok": true, + "expected_keys": [ + "confirmation_required" + ], + "given_keys": [ + "confirmation_required" + ] + } + ] + }, + { + "prompt": "I would like to upgrade the security features of my smart home and ensure everyt", + "reply_first_160": "\n\n\nfront door\n\n\nYRD256 Yale Assure Lock SL\n\n\n\n\nRelax Mode\n\n\nsoft glow\n\n\n\n\nsmart_plug_123\n\n\n19:00\n\n\n23:00\n<", + "expected_call": true, + "n_calls": 1, + "calls": [ + { + "name": "schedule_smart_plug", + "arguments": { + "device_id": "smart_plug_123", + "on_time": "19:00", + "off_time": "23:00", + "repeat_daily": "True" + } + } + ], + "scores": [ + { + "parse": true, + "name_ok": true, + "schema_ok": true, + "expected_keys": [ + "device_id", + "off_time", + "on_time", + "repeat_daily" + ], + "given_keys": [ + "device_id", + "off_time", + "on_time", + "repeat_daily" + ] + } + ] + }, + { + "prompt": "I'm conducting a research project where I aim to simulate one-dimensional period", + "reply_first_160": "\n\n\nperiodic\n\n\n5\n\n\n\n\n2\n\n\n00\n\n\n\n\n\n\n\n2\n\n\n{\"terms\": [{\"coeff\": 1, \"pauli_operators\": \"X\"}, {\"coe", + "expected_call": true, + "n_calls": 2, + "calls": [ + { + "name": "generate_quantum_circuit", + "arguments": { + "qubits": "2", + "hamiltonian": "{\"terms\": [{\"coeff\": 1, \"pauli_operators\": \"X\"}, {\"coeff\": 0.5, \"pauli_operators\": \"Z\"}]}", + "evolution_time": "1.0" + } + }, + { + "name": "perform_quantum_measurement", + "arguments": { + "circuit_id": "generated_circuit_id", + "measurement": "energy_distribution", + "shots": "1000" + } + } + ], + "scores": [ + { + "parse": true, + "name_ok": true, + "schema_ok": true, + "expected_keys": [ + "evolution_time", + "hamiltonian", + "qubits" + ], + "given_keys": [ + "evolution_time", + "hamiltonian", + "qubits" + ] + }, + { + "parse": true, + "name_ok": true, + "schema_ok": true, + "expected_keys": [ + "circuit_id", + "measurement", + "shots" + ], + "given_keys": [ + "circuit_id", + "measurement", + "shots" + ] + } + ] + }, + { + "prompt": "I am conducting a study on quantum mechanical simulations for chemical reactions", + "reply_first_160": "\n\n\nChemSimulationProject\n\n\nEast US\n\n\n\n\n1000\n\n\ntrue\n\n\n\n\n[\"Qiskit\", \"Cirq\", \"Q#\"]\n\n\n\n\n\n\n\ncybersecurity\n\n\nTrue\n\n\n\n\nQiskit\n\n\nA series of controlled NOT gates and Hada", + "expected_call": true, + "n_calls": 1, + "calls": [ + { + "name": "debug_quantum_circuit", + "arguments": { + "framework": "Qiskit", + "circuit_description": "A series of controlled NOT gates and Hadamard gates designed to create superpositions and entanglements among qubits, transitioning to an equal superposition state before measurement.", + "expected_output": "Equal probabilities among all possible quantum states.", + "observed_output": "Skewed probabilities with certain states having higher probabilities than anticipated.", + "error_types": "[\"qubit_calibration\", \"environmental_noise\", \"improper_gate_implementation\"]" + } + } + ], + "scores": [ + { + "parse": true, + "name_ok": true, + "schema_ok": true, + "expected_keys": [ + "circuit_description", + "error_types", + "expected_output", + "framework", + "observed_output" + ], + "given_keys": [ + "circuit_description", + "error_types", + "expected_output", + "framework", + "observed_output" + ] + } + ] + }, + { + "prompt": "I would like to secure a hotel room at the Grand Oak Hotel for an upcoming busin", + "reply_first_160": "\n\n\nGrand Oak Hotel\n\n\n2023-04-25\n\n\n\n\np123\n\n\nd456\n\n\n2023-05-15\n", + "expected_call": true, + "n_calls": 2, + "calls": [ + { + "name": "book_appointment", + "arguments": { + "patient_id": "p123", + "dentist_id": "d456", + "preferred_date": "2023-05-15", + "time_slot": "14:00" + } + }, + { + "name": "send_appointment_reminder", + "arguments": { + "appointment_id": "a789", + "reminder_time": "1 day" + } + } + ], + "scores": [ + { + "parse": true, + "name_ok": true, + "schema_ok": true, + "expected_keys": [ + "dentist_id", + "patient_id", + "preferred_date", + "time_slot" + ], + "given_keys": [ + "dentist_id", + "patient_id", + "preferred_date", + "time_slot" + ] + }, + { + "parse": true, + "name_ok": true, + "schema_ok": true, + "expected_keys": [ + "appointment_id", + "reminder_time" + ], + "given_keys": [ + "appointment_id", + "reminder_time" + ] + } + ] + }, + { + "prompt": "As the Financial Manager of XYZ Corp, I need assistance with efficiently managin", + "reply_first_160": "\n\n\nXYZ Corp\n\n\n[{\"date\": \"2023-04-03\", \"category\": \"Office Suppli", + "expected_call": true, + "n_calls": 3, + "calls": [ + { + "name": "record_expenses", + "arguments": { + "business_id": "XYZ Corp", + "expense_data": "[{\"date\": \"2023-04-03\", \"category\": \"Office Supplies\", \"amount\": 200.0, \"description\": \"Stationery and printing\"}, {\"date\": \"2023-04-05\", \"category\": \"Utilities\", \"amount\": 400.0, \"description\": \"Electricity and water bills\"}, {\"date\": \"2023-04-08\", \"category\": \"Travel\", \"amount\": 350.0, \"description\": \"Business trip transportation\"}, {\"date\": \"2023-04-15\", \"category\": \"Maintenance\", \"amount\": 150.0, \"description\": \"Office equipment repairs\"}]" + } + }, + { + "name": "process_invoices", + "arguments": { + "business_id": "XYZ Corp", + "invoice_data": "[{\"invoice_id\": \"INV-301\", \"client_id\": \"CLI-101\", \"amount\": 1200.0, \"due_date\": \"2023-05-01\"}, {\"invoice_id\": \"INV-302\", \"client_id\": \"CLI-102\", \"amount\": 800.0, \"due_date\": \"2023-05-10\"}, {\"invoice_id\": \"INV-303\", \"client_id\": \"CLI-103\", \"amount\": 1500.0, \"due_date\": \"2023-05-15\"}]" + } + }, + { + "name": "track_payments", + "arguments": { + "business_id": "XYZ Corp", + "payment_data": "[{\"payment_id\": \"PAY-101\", \"invoice_id\": \"INV-301\", \"amount\": 1200.0, \"date_received\": \"2023-04-20\"}, {\"payment_id\": \"PAY-102\", \"invoice_id\": \"INV-302\", \"amount\": 800.0, \"date_received\": \"2023-04-25\"}, {\"payment_id\": \"PAY-103\", \"invoice_id\": \"INV-303\", \"amount\": 1500.0, \"date_received\": \"2023-04-30\"}]" + } + } + ], + "scores": [ + { + "parse": true, + "name_ok": true, + "schema_ok": true, + "expected_keys": [ + "business_id", + "expense_data" + ], + "given_keys": [ + "business_id", + "expense_data" + ] + }, + { + "parse": true, + "name_ok": true, + "schema_ok": true, + "expected_keys": [ + "business_id", + "invoice_data" + ], + "given_keys": [ + "business_id", + "invoice_data" + ] + }, + { + "parse": true, + "name_ok": true, + "schema_ok": true, + "expected_keys": [ + "business_id", + "payment_data" + ], + "given_keys": [ + "business_id", + "payment_data" + ] + } + ] + }, + { + "prompt": "As a case manager for the firm, I'm currently handling a complex lawsuit identif", + "reply_first_160": "\n\n\nCASE-12345\n\n\n[{\"name\": \"Emma Johnson\", \"role\": \"Plaintiff\"}, {\"name", + "expected_call": true, + "n_calls": 3, + "calls": [ + { + "name": "categorize_parties", + "arguments": { + "case_id": "CASE-12345", + "parties": "[{\"name\": \"Emma Johnson\", \"role\": \"Plaintiff\"}, {\"name\": \"Liam Smith\", \"role\": \"Plaintiff\"}, {\"name\": \"Sophia Williams\", \"role\": \"Defendant\"}, {\"name\": \"Oliver Brown\", \"role\": \"Defendant\"}]" + } + }, + { + "name": "index_documents", + "arguments": { + "case_id": "CASE-12345", + "documents": "[{\"title\": \"Legal Briefs\", \"type\": \"Brief\"}, {\"title\": \"Witness Statements\", \"type\": \"Statement\"}, {\"title\": \"Court Orders\", \"type\": \"Order\"}, {\"title\": \"Evidence Submissions\", \"type\": \"Submission\"}]" + } + }, + { + "name": "setup_case_timeline", + "arguments": { + "case_id": "CASE-12345", + "milestones": "[{\"name\": \"Initial Hearing\", \"due_date\": \"2023-05-15\"}, {\"name\": \"Discovery Completion\", \"due_date\": \"2023-07-20\"}, {\"name\": \"Trial Start\", \"due_date\": \"2023-11-05\"}]" + } + } + ], + "scores": [ + { + "parse": true, + "name_ok": true, + "schema_ok": true, + "expected_keys": [ + "case_id", + "parties" + ], + "given_keys": [ + "case_id", + "parties" + ] + }, + { + "parse": true, + "name_ok": true, + "schema_ok": true, + "expected_keys": [ + "case_id", + "documents" + ], + "given_keys": [ + "case_id", + "documents" + ] + }, + { + "parse": true, + "name_ok": true, + "schema_ok": true, + "expected_keys": [ + "case_id", + "milestones" + ], + "given_keys": [ + "case_id", + "milestones" + ] + } + ] + }, + { + "prompt": "I am the HR manager at TechFusion Inc. and we're implementing new procedures to ", + "reply_first_160": "\n\n\nJames Ford\n\n\njames.ford@techfusion.com\n\n\n\n\n/path/to/patient/records\n\n\n\n\n\n\n\n[{\"client_id\": \"client_001\", \"services\": [{\"service_id\": \"service_001\", \"quantity\": 10, \"rate\":", + "expected_call": true, + "n_calls": 1, + "calls": [ + { + "name": "generate_invoices", + "arguments": { + "client_data": "[{\"client_id\": \"client_001\", \"services\": [{\"service_id\": \"service_001\", \"quantity\": 10, \"rate\": 250}, {\"service_id\": \"service_002\", \"quantity\": 5, \"rate\": 300}], \"billing_period\": \"April 1, 2023, to April 30, 2023\"}, {\"client_id\": \"client_002\", \"services\": [{\"service_id\": \"service_003\", \"quantity\": 15, \"rate\": 130}], \"billing_period\": \"April 1, 2023, to April 30, 2023\"}]", + "send_invoices": "True" + } + } + ], + "scores": [ + { + "parse": true, + "name_ok": true, + "schema_ok": true, + "expected_keys": [ + "client_data", + "send_invoices" + ], + "given_keys": [ + "client_data", + "send_invoices" + ] + } + ] + }, + { + "prompt": "As the operations manager of the Grandeur Hotel (ID: H1001), I'm focused on stre", + "reply_first_160": "\n\n\nH1001\n\n\n[\"Booking.com\", \"Expedia\", \"Airbnb\"]\n\n\n\nProj-456\n\n\nmaterials\n\n\n32", + "expected_call": true, + "n_calls": 13, + "calls": [ + { + "name": "record_project_expense", + "arguments": { + "project_id": "Proj-456", + "expense_type": "materials", + "amount": "3200", + "currency": "USD", + "date": "2023-04-05", + "vendor": "Timber Supplies Co.", + "description": "Purchase of lumber for construction materials" + } + }, + { + "name": "record_project_expense", + "arguments": { + "project_id": "Proj-456", + "expense_type": "materials", + "amount": "3200", + "currency": "USD", + "date": "2023-04-05", + "vendor": "Timber Supplies Co.", + "description": "Purchase of lumber for construction materials" + } + }, + { + "name": "record_project_expense", + "arguments": { + "project_id": "Proj-456", + "expense_type": "materials", + "amount": "3200", + "currency": "USD", + "date": "2023-04-05", + "vendor": "Timber Supplies Co.", + "description": "Purchase of lumber for construction materials" + } + }, + { + "name": "categorize_project_expense", + "arguments": { + "project_id": "Proj-456", + "expense_id": "Expense-1", + "category": "materials" + } + }, + { + "name": "categorize_project_expense", + "arguments": { + "project_id": "Proj-456", + "expense_id": "Expense-1", + "category": "materials" + } + }, + { + "name": "categorize_project_expense", + "arguments": { + "project_id": "Proj-456", + "expense_id": "Expense-1", + "category": "materials" + } + }, + { + "name": "generate_project_financial_report", + "arguments": { + "project_id": "Proj-456", + "report_type": "summary", + "date_range": "{\"start_date\": \"2023-04-01\", \"end_date\": \"2023-04-30\"}" + } + }, + { + "name": "generate_project_financial_report", + "arguments": { + "project_id": "Proj-456", + "report_type": "summary", + "date_range": "{\"start_date\": \"2023-04-01\", \"end_date\": \"2023-04-30\"}" + } + }, + { + "name": "generate_project_financial_report", + "arguments": { + "project_id": "Proj-456", + "report_type": "summary", + "date_range": "{\"start_date\": \"2023-04-01\", \"end_date\": \"2023-04-30\"}" + } + }, + { + "name": "generate_project_financial_report", + "arguments": { + "project_id": "Proj-456", + "report_type": "summary", + "date_range": "{\"start_date\": \"2023-04-01\", \"end_date\": \"2023-04-30\"}" + } + }, + { + "name": "generate_project_financial_report", + "arguments": { + "project_id": "Proj-456", + "report_type": "summary", + "date_range": "{\"start_date\": \"2023-04-01\", \"end_date\": \"2023-04-30\"}" + } + }, + { + "name": "generate_project_financial_report", + "arguments": { + "project_id": "Proj-456", + "report_type": "summary", + "date_range": "{\"start_date\": \"2023-04-01\", \"end_date\": \"2023-04-30\"}" + } + }, + { + "name": "generate_project_financial_report", + "arguments": { + "project_id": "Proj-456", + "report_type": "summary", + "date_range": "{\"start_date\": \"2023-04-01\", \"end_date\": \"2023-04-30\"}" + } + } + ], + "scores": [ + { + "parse": true, + "name_ok": true, + "schema_ok": true, + "expected_keys": [ + "amount", + "currency", + "date", + "description", + "expense_type", + "project_id", + "vendor" + ], + "given_keys": [ + "amount", + "currency", + "date", + "description", + "expense_type", + "project_id", + "vendor" + ] + }, + { + "parse": true, + "name_ok": true, + "schema_ok": true, + "expected_keys": [ + "amount", + "currency", + "date", + "description", + "expense_type", + "project_id", + "vendor" + ], + "given_keys": [ + "amount", + "currency", + "date", + "description", + "expense_type", + "project_id", + "vendor" + ] + }, + { + "parse": true, + "name_ok": true, + "schema_ok": true, + "expected_keys": [ + "amount", + "currency", + "date", + "description", + "expense_type", + "project_id", + "vendor" + ], + "given_keys": [ + "amount", + "currency", + "date", + "description", + "expense_type", + "project_id", + "vendor" + ] + }, + { + "parse": true, + "name_ok": true, + "schema_ok": true, + "expected_keys": [ + "category", + "expense_id", + "project_id" + ], + "given_keys": [ + "category", + "expense_id", + "project_id" + ] + }, + { + "parse": true, + "name_ok": true, + "schema_ok": true, + "expected_keys": [ + "category", + "expense_id", + "project_id" + ], + "given_keys": [ + "category", + "expense_id", + "project_id" + ] + }, + { + "parse": true, + "name_ok": true, + "schema_ok": true, + "expected_keys": [ + "category", + "expense_id", + "project_id" + ], + "given_keys": [ + "category", + "expense_id", + "project_id" + ] + }, + { + "parse": true, + "name_ok": true, + "schema_ok": true, + "expected_keys": [ + "date_range", + "project_id", + "report_type" + ], + "given_keys": [ + "date_range", + "project_id", + "report_type" + ] + }, + { + "parse": true, + "name_ok": true, + "schema_ok": true, + "expected_keys": [ + "date_range", + "project_id", + "report_type" + ], + "given_keys": [ + "date_range", + "project_id", + "report_type" + ] + }, + { + "parse": true, + "name_ok": true, + "schema_ok": true, + "expected_keys": [ + "date_range", + "project_id", + "report_type" + ], + "given_keys": [ + "date_range", + "project_id", + "report_type" + ] + }, + { + "parse": true, + "name_ok": true, + "schema_ok": true, + "expected_keys": [ + "date_range", + "project_id", + "report_type" + ], + "given_keys": [ + "date_range", + "project_id", + "report_type" + ] + }, + { + "parse": true, + "name_ok": true, + "schema_ok": true, + "expected_keys": [ + "date_range", + "project_id", + "report_type" + ], + "given_keys": [ + "date_range", + "project_id", + "report_type" + ] + }, + { + "parse": true, + "name_ok": true, + "schema_ok": true, + "expected_keys": [ + "date_range", + "project_id", + "report_type" + ], + "given_keys": [ + "date_range", + "project_id", + "report_type" + ] + }, + { + "parse": true, + "name_ok": true, + "schema_ok": true, + "expected_keys": [ + "date_range", + "project_id", + "report_type" + ], + "given_keys": [ + "date_range", + "project_id", + "report_type" + ] + } + ] + }, + { + "prompt": "I am managing the student database for a college and I require assistance with t", + "reply_first_160": "\n\n\nJohn Doe\n\n\n20\n\n\nComputer Science\n\n\n\n\n/path/to/emr/data\n\n\n[\"diagnosis\", \"treatment\", \"medication", + "expected_call": true, + "n_calls": 3, + "calls": [ + { + "name": "sort_records_by_category", + "arguments": { + "records": "/path/to/emr/data", + "categories": "[\"diagnosis\", \"treatment\", \"medication_history\"]" + } + }, + { + "name": "merge_duplicate_profiles", + "arguments": { + "records": "/path/to/emr/data" + } + }, + { + "name": "update_contact_information", + "arguments": { + "records": "/path/to/emr/data", + "contact_updates": "/path/to/contact/updates" + } + } + ], + "scores": [ + { + "parse": true, + "name_ok": true, + "schema_ok": true, + "expected_keys": [ + "categories", + "records" + ], + "given_keys": [ + "categories", + "records" + ] + }, + { + "parse": true, + "name_ok": true, + "schema_ok": true, + "expected_keys": [ + "records" + ], + "given_keys": [ + "records" + ] + }, + { + "parse": true, + "name_ok": true, + "schema_ok": true, + "expected_keys": [ + "contact_updates", + "records" + ], + "given_keys": [ + "contact_updates", + "records" + ] + } + ] + }, + { + "prompt": "As the Registrar at Prestige University, I am responsible for maintaining curren", + "reply_first_160": "\n\n\nS001234\n\n\neliza.ramirez@presti.ge\n\n\n\n\nCybersecurity Fundamentals\n\n\nAn in-depth training initiati", + "expected_call": true, + "n_calls": 1, + "calls": [ + { + "name": "schedule_training_program", + "arguments": { + "title": "Cybersecurity Fundamentals", + "description": "An in-depth training initiative designed to enhance the cybersecurity knowledge and skills of all employees.", + "modules": "[\"Introduction to Cybersecurity\", \"Phishing Detection Tactics\", \"Password Hygiene\", \"Data Protection Best Practices\"]", + "quizzes": "True", + "track_completion": "True" + } + } + ], + "scores": [ + { + "parse": true, + "name_ok": true, + "schema_ok": true, + "expected_keys": [ + "description", + "modules", + "quizzes", + "title", + "track_completion" + ], + "given_keys": [ + "description", + "modules", + "quizzes", + "title", + "track_completion" + ] + } + ] + }, + { + "prompt": "As the IT manager for our mid-sized manufacturing company, I have collected the ", + "reply_first_160": "\n\n\n{\"products\": [{\"id\": \"P001\", \"name\": \"Product A\", \"stock\": 500, \"supplier\": \"Su", + "expected_call": true, + "n_calls": 3, + "calls": [ + { + "name": "integrate_inventory_management", + "arguments": { + "inventory_data": "{\"products\": [{\"id\": \"P001\", \"name\": \"Product A\", \"stock\": 500, \"supplier\": \"Supplier X\"}, {\"id\": \"P002\", \"name\": \"Product B\", \"stock\": 300, \"supplier\": \"Supplier Y\"}], \"warehouses\": [{\"id\": \"W001\", \"location\": \"Warehouse A\", \"capacity\": 1000}, {\"id\": \"W002\", \"location\": \"Warehouse B\", \"capacity\": 800}]}", + "erp_system_endpoint": "https://erp.ourcompany.com/api/inventory_integration" + } + }, + { + "name": "integrate_accounting_data", + "arguments": { + "accounting_data": "{\"transactions\": [{\"id\": \"T001\", \"type\": \"income\", \"amount\": 5000, \"category\": \"Sales Revenue\"}, {\"id\": \"T002\", \"type\": \"expense\", \"amount\": 2000, \"category\": \"Raw Materials\"}], \"invoices\": [{\"id\": \"INV001\", \"customer\": \"Customer A\", \"amount\": 1000, \"status\": \"paid\"}, {\"id\": \"INV002\", \"customer\": \"Customer B\", \"amount\": 1500, \"status\": \"pending\"}]}", + "erp_system_endpoint": "https://erp.ourcompany.com/api/accounting_data_integration" + } + }, + { + "name": "integrate_crm_data", + "arguments": { + "crm_data": "{\"clients\": [{\"id\": \"C001\", \"name\": \"Client A\", \"contact\": \"John Doe\", \"email\": \"john.doe@clienta.com\"}, {\"id\": \"C002\", \"name\": \"Client B\", \"contact\": \"Jane Smith\", \"email\": \"jane.smith@clientb.com\"}], \"sales_pipelines\": [{\"id\": \"SP001\", \"client\": \"Client A\", \"stage\": \"Proposal Sent\", \"value\": 5000}, {\"id\": \"SP002\", \"client\": \"Client B\", \"stage\": \"Negotiation\", \"value\": 7500}]}", + "erp_system_endpoint": "https://erp.ourcompany.com/api/crm_data_integration" + } + } + ], + "scores": [ + { + "parse": true, + "name_ok": true, + "schema_ok": true, + "expected_keys": [ + "erp_system_endpoint", + "inventory_data" + ], + "given_keys": [ + "erp_system_endpoint", + "inventory_data" + ] + }, + { + "parse": true, + "name_ok": true, + "schema_ok": true, + "expected_keys": [ + "accounting_data", + "erp_system_endpoint" + ], + "given_keys": [ + "accounting_data", + "erp_system_endpoint" + ] + }, + { + "parse": true, + "name_ok": true, + "schema_ok": true, + "expected_keys": [ + "crm_data", + "erp_system_endpoint" + ], + "given_keys": [ + "crm_data", + "erp_system_endpoint" + ] + } + ] + }, + { + "prompt": "I am undertaking a comprehensive analysis of my investment portfolio which inclu", + "reply_first_160": "\n\n\n[\"AAPL\", \"MSFT\", \"GOOGL\"]\n\n\n2020-01-01\n\n<", + "expected_call": true, + "n_calls": 3, + "calls": [ + { + "name": "fetch_historical_stock_prices", + "arguments": { + "tickers": "[\"AAPL\", \"MSFT\", \"GOOGL\"]", + "start_date": "2020-01-01", + "end_date": "2023-01-01" + } + }, + { + "name": "calculate_future_returns", + "arguments": { + "tickers": "[\"AAPL\", \"MSFT\", \"GOOGL\"]", + "scenarios": "[{\"name\": \"Optimistic\", \"growth_rate\": 0.15}, {\"name\": \"Pessimistic\", \"growth_rate\": -0.025}, {\"name\": \"Neutral\", \"growth_rate\": 0.06}]" + } + }, + { + "name": "assess_portfolio_risk", + "arguments": { + "portfolio": "[{\"ticker\": \"AAPL\", \"weight\": 0.4}, {\"ticker\": \"MSFT\", \"weight\": 0.35}, {\"ticker\": \"GOOGL\", \"weight\": 0.25}]", + "risk_model": "VaR" + } + } + ], + "scores": [ + { + "parse": true, + "name_ok": true, + "schema_ok": true, + "expected_keys": [ + "end_date", + "start_date", + "tickers" + ], + "given_keys": [ + "end_date", + "start_date", + "tickers" + ] + }, + { + "parse": true, + "name_ok": true, + "schema_ok": true, + "expected_keys": [ + "scenarios", + "tickers" + ], + "given_keys": [ + "scenarios", + "tickers" + ] + }, + { + "parse": true, + "name_ok": true, + "schema_ok": true, + "expected_keys": [ + "portfolio", + "risk_model" + ], + "given_keys": [ + "portfolio", + "risk_model" + ] + } + ] + }, + { + "prompt": "I am part of a legal team managing an upcoming case identified as \"Case-001\" and", + "reply_first_160": "\n\n\nCase-001\n\n\n[{\"name\": \"Initial Client Meeting\", \"du", + "expected_call": true, + "n_calls": 1, + "calls": [ + { + "name": "initialize_workflow_organization", + "arguments": { + "case_id": "Case-001", + "milestones": "[{\"name\": \"Initial Client Meeting\", \"due_date\": \"2023-05-10\"}, {\"name\": \"Discovery Phase Completion\", \"due_date\": \"2023-06-15\"}, {\"name\": \"Pre-trial Motions\", \"due_date\": \"2023-08-01\"}]", + "document_deadlines": "[{\"document_name\": \"Client Affidavit\", \"due_date\": \"2023-05-20\"}, {\"document_name\": \"Witness List\", \"due_date\": \"2023-07-05\"}]", + "team_members": "[{\"name\": \"John Doe\", \"role\": \"Lead Attorney\", \"tasks\": [\"case strategy\", \"client communications\"]}, {\"name\": \"Jane Smith\", \"role\": \"Paralegal\", \"tasks\": [\"research\", \"documentation\"]}, {\"name\": \"Richard Roe\", \"role\": \"Junior Attorney\", \"tasks\": [\"assisting with motions\", \"legal analysis\"]}, {\"name\": \"Emily Davis\", \"role\": \"Legal Secretary\", \"tasks\": [\"scheduling\", \"filing\"]}]" + } + } + ], + "scores": [ + { + "parse": true, + "name_ok": true, + "schema_ok": true, + "expected_keys": [ + "case_id", + "document_deadlines", + "milestones", + "team_members" + ], + "given_keys": [ + "case_id", + "document_deadlines", + "milestones", + "team_members" + ] + } + ] + }, + { + "prompt": "As a project manager at Delphi Consulting Group, I am currently spearheading the", + "reply_first_160": "\n\n\nReengineering the finance and accounting workflow for Apex Global Solutions.\n\n\n\n\nsmart lock\n\n\nSecureLock SLT2\n\n\n\n\n\n12345\n\n\nTrue\n\n\n", + "expected_call": true, + "n_calls": 1, + "calls": [ + { + "name": "getConnectedDevicesStatus", + "arguments": { + "locationId": "12345", + "includeOffline": "True" + } + } + ], + "scores": [ + { + "parse": true, + "name_ok": true, + "schema_ok": true, + "expected_keys": [ + "includeOffline", + "locationId" + ], + "given_keys": [ + "includeOffline", + "locationId" + ] + } + ] + }, + { + "prompt": "I am currently deploying a smart building management system and need to integrat", + "reply_first_160": "\n\n\nEnviroSensorType\n\n\nEnvironmentalSensor_001\n\n\n\n\nsensor-12345\n\n\nTemperature Sensor\n\n\n\n\nsmart-city-123\n\n\nus-central1\n\n\n\n\ntempSensor_001\n\n\nTemperature Sensor 001\n\n\n\n\nhome_network_123\n\n\n\n\n\n\n\n9BX5Z6R7TY101FY2\n\n\n1357924\n\n\n\n\nEco-Friendly Water Bottle\n\n\n[{\"demographic\": \"young adul", + "expected_call": true, + "n_calls": 1, + "calls": [ + { + "name": "create_campaign", + "arguments": { + "product_name": "Eco-Friendly Water Bottle", + "campaign_details": "[{\"demographic\": \"young adults\", \"platforms\": [\"Instagram\", \"Snapchat\"], \"budget\": 5000, \"start_date\": \"2023-07-01\", \"end_date\": \"2023-07-15\", \"ad_schedule\": [{\"ad_name\": \"EcoLaunch Day\", \"release_date\": \"2023-07-01\"}, {\"ad_name\": \"EcoHydrate Promo\", \"release_date\": \"2023-07-05\"}, {\"ad_name\": \"EcoWeekend Special\", \"release_date\": \"2023-07-08\"}]}, {\"demographic\": \"professionals\", \"platforms\": [\"LinkedIn\", \"Twitter\"], \"budget\": 8000, \"start_date\": \"2023-07-01\", \"end_date\": \"2023-07-15\", \"ad_schedule\": [{\"ad_name\": \"EcoWork Week\", \"release_date\": \"2023-07-01\"}, {\"ad_name\": \"EcoCommute Deal\", \"release_date\": \"2023-07-05\"}, {\"ad_name\": \"EcoEndOfWeek\", \"release_date\": \"2023-07-08\"}]}]" + } + } + ], + "scores": [ + { + "parse": true, + "name_ok": true, + "schema_ok": true, + "expected_keys": [ + "campaign_details", + "product_name" + ], + "given_keys": [ + "campaign_details", + "product_name" + ] + } + ] + }, + { + "prompt": "Our marketing team has conducted three digital advertising campaigns identified ", + "reply_first_160": "\n\n\n[\"camp_001\", \"camp_002\", \"camp_003\"]\n\n\n[\"impressions\", \"clic", + "expected_call": true, + "n_calls": 2, + "calls": [ + { + "name": "get_campaign_metrics", + "arguments": { + "campaign_ids": "[\"camp_001\", \"camp_002\", \"camp_003\"]", + "metrics": "[\"impressions\", \"click_through_rate\", \"conversion_rate\"]", + "date_range": "{\"start_date\": \"2023-01-01\", \"end_date\": \"2023-03-31\"}" + } + }, + { + "name": "calculate_campaign_roi", + "arguments": { + "campaign_ids": "[\"camp_001\", \"camp_002\", \"camp_003\"]", + "costs": "[20000, 25000, 15000]" + } + } + ], + "scores": [ + { + "parse": true, + "name_ok": true, + "schema_ok": true, + "expected_keys": [ + "campaign_ids", + "date_range", + "metrics" + ], + "given_keys": [ + "campaign_ids", + "date_range", + "metrics" + ] + }, + { + "parse": true, + "name_ok": true, + "schema_ok": true, + "expected_keys": [ + "campaign_ids", + "costs" + ], + "given_keys": [ + "campaign_ids", + "costs" + ] + } + ] + }, + { + "prompt": "As an avid movie buff, I have curated a digital collection on my home entertainm", + "reply_first_160": "\n\n\nuser123\n\n\n[{'content_id': 'movie_201", + "expected_call": true, + "n_calls": 3, + "calls": [ + { + "name": "recommend_content_based_on_viewing_habits", + "arguments": { + "user_id": "user123", + "viewing_history": "[{'content_id': 'movie_201', 'watched_on': '2023-04-14T20:00:00Z'}, {'content_id': 'comedy_789', 'watched_on': '2023-04-16T18:30:00Z'}]", + "genre_preferences": "['action', 'drama']" + } + }, + { + "name": "adjust_settings_to_user_preferences", + "arguments": { + "user_id": "user123", + "settings_preferences": "{\"audio_level\": 50, \"subtitle_language\": \"English\", \"display_brightness\": 70}" + } + }, + { + "name": "set_parental_control_options", + "arguments": { + "user_id": "user123", + "parental_settings": "{\"enabled\": true, \"rating_limit\": \"PG-13\"}" + } + } + ], + "scores": [ + { + "parse": true, + "name_ok": true, + "schema_ok": true, + "expected_keys": [ + "genre_preferences", + "user_id", + "viewing_history" + ], + "given_keys": [ + "genre_preferences", + "user_id", + "viewing_history" + ] + }, + { + "parse": true, + "name_ok": true, + "schema_ok": true, + "expected_keys": [ + "settings_preferences", + "user_id" + ], + "given_keys": [ + "settings_preferences", + "user_id" + ] + }, + { + "parse": true, + "name_ok": true, + "schema_ok": true, + "expected_keys": [ + "parental_settings", + "user_id" + ], + "given_keys": [ + "parental_settings", + "user_id" + ] + } + ] + }, + { + "prompt": "I am setting up an interactive home entertainment interface and need to configur", + "reply_first_160": "\n\n\nhome_theater_001\n\n\nMovie Night\n\n\n\n\n\n[\"router_01\", \"switch_02\", \"modem_03\"]\n\n\n\n", + "expected_call": true, + "n_calls": 2, + "calls": [ + { + "name": "get_network_device_status", + "arguments": { + "device_ids": "[\"router_01\", \"switch_02\", \"modem_03\"]" + } + }, + { + "name": "analyze_network_traffic", + "arguments": { + "time_range": "last_24_hours", + "traffic_type": "all" + } + } + ], + "scores": [ + { + "parse": true, + "name_ok": true, + "schema_ok": true, + "expected_keys": [ + "device_ids" + ], + "given_keys": [ + "device_ids" + ] + }, + { + "parse": true, + "name_ok": true, + "schema_ok": true, + "expected_keys": [ + "time_range", + "traffic_type" + ], + "given_keys": [ + "time_range", + "traffic_type" + ] + } + ] + }, + { + "prompt": "During peak hours, my streaming service is facing issues with video buffering, a", + "reply_first_160": "\n\n\nabc12345\n\n\npeak_hours\n\n\n\n\n\nmedia_content_123\n\n\n[\"free\", \"premium\"]\n\n\n\n\nNinaSimone\n\n\nSunsetMelodies2023\n\n\n\n\nSaturday Night Trivia\n\n\nAlex Johnson\n\n", + "player_emails": "[\"jessica.smith@email.com\", \"david.brown@email.com\", \"emma.williams@email.com\", \"michael.jones@email.com\"]" + } + } + ], + "scores": [ + { + "parse": true, + "name_ok": true, + "schema_ok": true, + "expected_keys": [ + "categories", + "difficulty", + "game_title", + "host_name", + "question_count" + ], + "given_keys": [ + "categories", + "difficulty", + "game_title", + "host_name", + "question_count" + ] + }, + { + "parse": true, + "name_ok": true, + "schema_ok": true, + "expected_keys": [ + "player_emails", + "session_id" + ], + "given_keys": [ + "player_emails", + "session_id" + ] + } + ] + }, + { + "prompt": "I'm tasked with enhancing the security of our telecommunications network 'net_00", + "reply_first_160": "\n\n\nnet_001\n\n\nAES-256\n\n\n\n\nNatureFan88\n\n\nEcoLover2023!\n\n\n\n", + "expected_call": true, + "n_calls": 2, + "calls": [ + { + "name": "authenticate_user", + "arguments": { + "username": "NatureFan88", + "password": "EcoLover2023!" + } + }, + { + "name": "stream_episode", + "arguments": { + "user_token": "[User Token]", + "show_id": "pe2_show_001", + "episode_number": "6", + "device_id": "device_987654321" + } + } + ], + "scores": [ + { + "parse": true, + "name_ok": true, + "schema_ok": true, + "expected_keys": [ + "password", + "username" + ], + "given_keys": [ + "password", + "username" + ] + }, + { + "parse": true, + "name_ok": true, + "schema_ok": true, + "expected_keys": [ + "device_id", + "episode_number", + "show_id", + "user_token" + ], + "given_keys": [ + "device_id", + "episode_number", + "show_id", + "user_token" + ] + } + ] + }, + { + "prompt": "I would like to enhance our telecom CRM system with several updates. Firstly, I ", + "reply_first_160": "\n\n\nJD_8098\n\n\n{\"address\": \"1234 Telecom Lane, Datacity", + "expected_call": true, + "n_calls": 4, + "calls": [ + { + "name": "update_customer_profile", + "arguments": { + "customer_id": "JD_8098", + "profile_updates": "{\"address\": \"1234 Telecom Lane, Datacity, DC 90909\", \"email\": \"john.doe@email.com\", \"phone_plan\": \"Unlimited Data Plan\"}" + } + }, + { + "name": "track_service_usage", + "arguments": { + "customer_id": "JS_1002", + "start_date": "2023-03-01", + "end_date": "2023-03-31" + } + }, + { + "name": "manage_support_tickets", + "arguments": { + "ticket_id": "TKT_4321", + "status": "Resolved", + "resolution_notes": "Replaced faulty router, service restored" + } + }, + { + "name": "analyze_customer_feedback", + "arguments": { + "feedback_data": "{\"feedbacks\": [{\"id\": \"FB_001\", \"customer_id\": \"JD_8098\", \"rating\": 5, \"comment\": \"Great service!\"}, {\"id\": \"FB_002\", \"customer_id\": \"JS_1002\", \"rating\": 4, \"comment\": \"Good coverage, but slow speeds sometimes.\"}]}" + } + } + ], + "scores": [ + { + "parse": true, + "name_ok": true, + "schema_ok": true, + "expected_keys": [ + "customer_id", + "profile_updates" + ], + "given_keys": [ + "customer_id", + "profile_updates" + ] + }, + { + "parse": true, + "name_ok": true, + "schema_ok": true, + "expected_keys": [ + "customer_id", + "end_date", + "start_date" + ], + "given_keys": [ + "customer_id", + "end_date", + "start_date" + ] + }, + { + "parse": true, + "name_ok": true, + "schema_ok": true, + "expected_keys": [ + "resolution_notes", + "status", + "ticket_id" + ], + "given_keys": [ + "resolution_notes", + "status", + "ticket_id" + ] + }, + { + "parse": true, + "name_ok": true, + "schema_ok": true, + "expected_keys": [ + "feedback_data" + ], + "given_keys": [ + "feedback_data" + ] + } + ] + }, + { + "prompt": "I am planning to host 'Ultimate Trivia Challenge,' a virtual game night for me a", + "reply_first_160": "\n\n\nUltimate Trivia Challenge\n\n\n5\n\n\n\n\nSAT-001\n\n\n10\n\n\n", + "expected_call": true, + "n_calls": 3, + "calls": [ + { + "name": "track_satellite_position", + "arguments": { + "satellite_id": "SAT-001", + "time_interval": "10" + } + }, + { + "name": "allocate_bandwidth", + "arguments": { + "channel_id": "CH-101", + "bandwidth_mbps": "50" + } + }, + { + "name": "detect_network_fault", + "arguments": { + "network_segment": "Segment-3", + "detection_threshold": "100" + } + } + ], + "scores": [ + { + "parse": true, + "name_ok": true, + "schema_ok": true, + "expected_keys": [ + "satellite_id", + "time_interval" + ], + "given_keys": [ + "satellite_id", + "time_interval" + ] + }, + { + "parse": true, + "name_ok": true, + "schema_ok": true, + "expected_keys": [ + "bandwidth_mbps", + "channel_id" + ], + "given_keys": [ + "bandwidth_mbps", + "channel_id" + ] + }, + { + "parse": true, + "name_ok": true, + "schema_ok": true, + "expected_keys": [ + "detection_threshold", + "network_segment" + ], + "given_keys": [ + "detection_threshold", + "network_segment" + ] + } + ] + }, + { + "prompt": "I'm leading the analytics for our multi-channel advertising campaign, and I have", + "reply_first_160": "\n\n\nmulti_channel_campaign\n\n\n[\"Facebook\", \"Google\", \"Instagram\"", + "expected_call": true, + "n_calls": 3, + "calls": [ + { + "name": "extract_campaign_data", + "arguments": { + "campaign_id": "multi_channel_campaign", + "channels": "[\"Facebook\", \"Google\", \"Instagram\"]", + "metrics": "[\"cost\", \"impressions\", \"clicks\", \"conversions\"]" + } + }, + { + "name": "calculate_cost_effectiveness", + "arguments": { + "campaign_data": "{\"Facebook\": {\"cost\": 1200, \"impressions\": 30000, \"clicks\": 1500, \"conversions\": 150}, \"Google\": {\"cost\": 1000, \"impressions\": 25000, \"clicks\": 1800, \"conversions\": 200}, \"Instagram\": {\"cost\": 800, \"impressions\": 20000, \"clicks\": 1000, \"conversions\": 100}}" + } + }, + { + "name": "analyze_audience_engagement", + "arguments": { + "campaign_data": "{\"Facebook\": {\"cost\": 1200, \"impressions\": 30000, \"clicks\": 1500, \"conversions\": 150}, \"Google\": {\"cost\": 1000, \"impressions\": 25000, \"clicks\": 1800, \"conversions\": 200}, \"Instagram\": {\"cost\": 800, \"impressions\": 20000, \"clicks\": 1000, \"conversions\": 100}}" + } + } + ], + "scores": [ + { + "parse": true, + "name_ok": true, + "schema_ok": true, + "expected_keys": [ + "campaign_id", + "channels", + "metrics" + ], + "given_keys": [ + "campaign_id", + "channels", + "metrics" + ] + }, + { + "parse": true, + "name_ok": true, + "schema_ok": true, + "expected_keys": [ + "campaign_data" + ], + "given_keys": [ + "campaign_data" + ] + }, + { + "parse": true, + "name_ok": true, + "schema_ok": true, + "expected_keys": [ + "campaign_data" + ], + "given_keys": [ + "campaign_data" + ] + } + ] + }, + { + "prompt": "I have recently purchased a smart speaker, and I would like to integrate it into", + "reply_first_160": "\n\n\nSPK12345X\n\n\nSYS67890H\n\n\n\n\nCRM\n\n\n[\"age\", \"location\", \"product_preferences\"", + "expected_call": true, + "n_calls": 3, + "calls": [ + { + "name": "analyze_customer_data", + "arguments": { + "customer_data_source": "CRM", + "data_fields": "[\"age\", \"location\", \"product_preferences\", \"purchasing_frequency\"]" + } + }, + { + "name": "segment_audience", + "arguments": { + "analysis_results": "CRM", + "segmentation_criteria": "[\"age_ranges\", \"interests\", \"shopping_behavior\"]" + } + }, + { + "name": "create_targeted_ad_campaign", + "arguments": { + "audience_segments": "CRM", + "campaign_parameters": "{\"budget\": 50000, \"start_date\": \"2023-04-01\", \"end_date\": \"2023-06-30\", \"ad_creatives\": [\"video\", \"image\", \"text\"]}" + } + } + ], + "scores": [ + { + "parse": true, + "name_ok": true, + "schema_ok": true, + "expected_keys": [ + "customer_data_source", + "data_fields" + ], + "given_keys": [ + "customer_data_source", + "data_fields" + ] + }, + { + "parse": true, + "name_ok": true, + "schema_ok": true, + "expected_keys": [ + "analysis_results", + "segmentation_criteria" + ], + "given_keys": [ + "analysis_results", + "segmentation_criteria" + ] + }, + { + "parse": true, + "name_ok": true, + "schema_ok": true, + "expected_keys": [ + "audience_segments", + "campaign_parameters" + ], + "given_keys": [ + "audience_segments", + "campaign_parameters" + ] + } + ] + }, + { + "prompt": "I am arranging a virtual conference that requires an interactive media collabora", + "reply_first_160": "\n\n\n[\"video_conferencing\", \"real_time_editing\", \"progress_sharing\"]\n<", + "expected_call": true, + "n_calls": 4, + "calls": [ + { + "name": "find_interactive_media_collaboration_tool", + "arguments": { + "features_required": "[\"video_conferencing\", \"real_time_editing\", \"progress_sharing\"]", + "participant_limit": "100" + } + }, + { + "name": "setup_interactive_media_collaboration", + "arguments": { + "tool_name": "Zoom", + "conference_details": "{\"agenda\": \"Explore the latest trends in interactive media\", \"date_time\": \"2023-06-15T10:00:00Z\", \"participants\": [\"participant1@example.com\", \"participant2@example.com\"]}" + } + }, + { + "name": "setup_interactive_media_collaboration", + "arguments": { + "tool_name": "Zoom", + "conference_details": "{\"agenda\": \"Explore the latest trends in interactive media\", \"date_time\": \"2023-06-15T10:00:00Z\", \"participants\": [\"participant1@example.com\", \"participant2@example.com\"]}" + } + }, + { + "name": "setup_interactive_media_collaboration", + "arguments": { + "tool_name": "Zoom", + "conference_details": "{\"agenda\": \"Explore the latest trends in interactive media\", \"date_time\": \"2023-06-15T10:00:00Z\", \"participants\": [\"participant1@example.com\", \"participant2@example.com\"]}" + } + } + ], + "scores": [ + { + "parse": true, + "name_ok": true, + "schema_ok": true, + "expected_keys": [ + "features_required", + "participant_limit" + ], + "given_keys": [ + "features_required", + "participant_limit" + ] + }, + { + "parse": true, + "name_ok": true, + "schema_ok": true, + "expected_keys": [ + "conference_details", + "tool_name" + ], + "given_keys": [ + "conference_details", + "tool_name" + ] + }, + { + "parse": true, + "name_ok": true, + "schema_ok": true, + "expected_keys": [ + "conference_details", + "tool_name" + ], + "given_keys": [ + "conference_details", + "tool_name" + ] + }, + { + "parse": true, + "name_ok": true, + "schema_ok": true, + "expected_keys": [ + "conference_details", + "tool_name" + ], + "given_keys": [ + "conference_details", + "tool_name" + ] + } + ] + }, + { + "prompt": "For the \"Soundwave\" music festival organized by \"Harmony Events,\" taking place o", + "reply_first_160": "\n\n\nSoundwave\n\n\n[{\"artist_id\": \"A123\", \"stage\": \"Main Stage\", \"pe", + "expected_call": true, + "n_calls": 3, + "calls": [ + { + "name": "manage_artist_lineup", + "arguments": { + "festival_id": "Soundwave", + "artists": "[{\"artist_id\": \"A123\", \"stage\": \"Main Stage\", \"performance_time\": \"2023-07-15T16:00:00\"}, {\"artist_id\": \"B456\", \"stage\": \"Acoustic Stage\", \"performance_time\": \"2023-07-15T18:00:00\"}, {\"artist_id\": \"C789\", \"stage\": \"Main Stage\", \"performance_time\": \"2023-07-15T20:00:00\"}]" + } + }, + { + "name": "track_ticket_sales", + "arguments": { + "festival_id": "Soundwave", + "date": "2023-04-30" + } + }, + { + "name": "coordinate_event_logistics", + "arguments": { + "festival_id": "Soundwave", + "logistics_details": "{\"security_staff\": 50, \"medical_staff\": 20, \"parking_spots\": 3000}" + } + } + ], + "scores": [ + { + "parse": true, + "name_ok": true, + "schema_ok": true, + "expected_keys": [ + "artists", + "festival_id" + ], + "given_keys": [ + "artists", + "festival_id" + ] + }, + { + "parse": true, + "name_ok": true, + "schema_ok": true, + "expected_keys": [ + "date", + "festival_id" + ], + "given_keys": [ + "date", + "festival_id" + ] + }, + { + "parse": true, + "name_ok": true, + "schema_ok": true, + "expected_keys": [ + "festival_id", + "logistics_details" + ], + "given_keys": [ + "festival_id", + "logistics_details" + ] + } + ] + }, + { + "prompt": "Our marketing team has recently concluded several advertising campaigns across d", + "reply_first_160": "\n\n\n[\"ad_9801\", \"ad_7623\", \"ad_3472\"]\n\n\n[\"cost_per_acquisi", + "expected_call": true, + "n_calls": 1, + "calls": [ + { + "name": "aggregate_campaign_metrics", + "arguments": { + "campaign_ids": "[\"ad_9801\", \"ad_7623\", \"ad_3472\"]", + "metrics": "[\"cost_per_acquisition\", \"click_through_rate\", \"return_on_investment\"]", + "date_range": "{\"start_date\": \"2023-01-01\", \"end_date\": \"2023-03-31\"}" + } + } + ], + "scores": [ + { + "parse": true, + "name_ok": true, + "schema_ok": true, + "expected_keys": [ + "campaign_ids", + "date_range", + "metrics" + ], + "given_keys": [ + "campaign_ids", + "date_range", + "metrics" + ] + } + ] + }, + { + "prompt": "I am in charge of a live concert event scheduled for June 15, 2023, at the Madis", + "reply_first_160": "\n\n\n20230615_MADISON_SQUARE_GARDEN_CONCERT\n\n\nSW123456\n\n\n\n[{\"latitude\": -23.698, \"longitude\": 133.8807}, {\"latitude\": -13.163", + "expected_call": true, + "n_calls": 3, + "calls": [ + { + "name": "determine_optimal_satellite_coverage", + "arguments": { + "location_coordinates": "[{\"latitude\": -23.698, \"longitude\": 133.8807}, {\"latitude\": -13.1631, \"longitude\": -72.545}]", + "required_reliability": "99.5" + } + }, + { + "name": "estimate_required_bandwidth", + "arguments": { + "data_transfer_rate": "700", + "voice_channels": "20", + "video_conferencing": "True" + } + }, + { + "name": "select_satellite_service_provider", + "arguments": { + "coverage_area": "regional", + "bandwidth_requirement": "700", + "budget": "500000" + } + } + ], + "scores": [ + { + "parse": true, + "name_ok": true, + "schema_ok": true, + "expected_keys": [ + "location_coordinates", + "required_reliability" + ], + "given_keys": [ + "location_coordinates", + "required_reliability" + ] + }, + { + "parse": true, + "name_ok": true, + "schema_ok": true, + "expected_keys": [ + "data_transfer_rate", + "video_conferencing", + "voice_channels" + ], + "given_keys": [ + "data_transfer_rate", + "video_conferencing", + "voice_channels" + ] + }, + { + "parse": true, + "name_ok": true, + "schema_ok": true, + "expected_keys": [ + "bandwidth_requirement", + "budget", + "coverage_area" + ], + "given_keys": [ + "bandwidth_requirement", + "budget", + "coverage_area" + ] + } + ] + }, + { + "prompt": "Recently, we've observed a decrease in the quality of our telecommunications ser", + "reply_first_160": "\n\n\n123\n\n\nlast_24_hours\n\n\n", + "expected_call": true, + "n_calls": 2, + "calls": [ + { + "name": "analyze_signal_strength", + "arguments": { + "sector_id": "123", + "time_frame": "last_24_hours" + } + }, + { + "name": "assess_network_traffic", + "arguments": { + "network_id": "456", + "time_frame": "last_24_hours" + } + } + ], + "scores": [ + { + "parse": true, + "name_ok": true, + "schema_ok": true, + "expected_keys": [ + "sector_id", + "time_frame" + ], + "given_keys": [ + "sector_id", + "time_frame" + ] + }, + { + "parse": true, + "name_ok": true, + "schema_ok": true, + "expected_keys": [ + "network_id", + "time_frame" + ], + "given_keys": [ + "network_id", + "time_frame" + ] + } + ] + }, + { + "prompt": "I would like to enhance our Interactive Home Entertainment User Feedback System ", + "reply_first_160": "\n\n\n12345\n\n\n67890\n\n\n4\n\n\n\n\naction\n\n\nthis month\n\n\nrating\n\n\n\n123456789\n\n\n[\"sci-fi\", \"comedy\"]\n\n\n\n[\"quiz\", \"poll\", \"game\"]\n\n\nTrue\n\n\n\n\n{\"start_date\": \"2023-03-01\", \"end_date\": \"2023-03-31\"}\n\n\n\n\n[\"cmp_123\", \"cmp_456\"]\n\n\n[\"click_through_rate\", \"conve", + "expected_call": true, + "n_calls": 1, + "calls": [ + { + "name": "get_ad_performance_metrics", + "arguments": { + "campaign_ids": "[\"cmp_123\", \"cmp_456\"]", + "metrics": "[\"click_through_rate\", \"conversion_rate\", \"return_on_ad_spend\"]", + "date_range": "{\"start_date\": \"2023-01-01\", \"end_date\": \"2023-03-31\"}" + } + } + ], + "scores": [ + { + "parse": true, + "name_ok": true, + "schema_ok": true, + "expected_keys": [ + "campaign_ids", + "date_range", + "metrics" + ], + "given_keys": [ + "campaign_ids", + "date_range", + "metrics" + ] + } + ] + }, + { + "prompt": "I recently encountered an issue with my home entertainment system where voice co", + "reply_first_160": "\n\n\nHES12345678\n\n\n[2023-04-01T10:15:30Z] ERROR: Voice re", + "expected_call": true, + "n_calls": 2, + "calls": [ + { + "name": "diagnose_voice_command_issue", + "arguments": { + "system_id": "HES12345678", + "error_logs": "[2023-04-01T10:15:30Z] ERROR: Voice recognition failed to initiate.\n[2023-04-01T10:16:05Z] WARNING: Microphone input level low.\n[2023-04-01T10:17:10Z] ERROR: Streaming service 'Netflix' connection timed out." + } + }, + { + "name": "troubleshoot_streaming_service", + "arguments": { + "service_name": "Netflix", + "system_id": "HES12345678", + "user_id": "User78910" + } + } + ], + "scores": [ + { + "parse": true, + "name_ok": true, + "schema_ok": true, + "expected_keys": [ + "error_logs", + "system_id" + ], + "given_keys": [ + "error_logs", + "system_id" + ] + }, + { + "parse": true, + "name_ok": true, + "schema_ok": true, + "expected_keys": [ + "service_name", + "system_id", + "user_id" + ], + "given_keys": [ + "service_name", + "system_id", + "user_id" + ] + } + ] + }, + { + "prompt": "I would like to initiate an advertising campaign for our annual Summer Sale even", + "reply_first_160": "\n\n\nSummer Sale\n\n\n[\"Google Ads\", \"Facebook Ads\", \"Twitter Ads\"]", + "expected_call": true, + "n_calls": 2, + "calls": [ + { + "name": "create_ad_campaign", + "arguments": { + "campaign_name": "Summer Sale", + "platforms": "[\"Google Ads\", \"Facebook Ads\", \"Twitter Ads\"]", + "budget": "10000", + "start_date": "2023-06-01", + "end_date": "2023-07-31", + "target_audience": "{\"location\": \"United States\", \"interests\": [\"hiking\", \"camping\", \"beach-related recreation\"], \"age_range\": [25, 40]}" + } + }, + { + "name": "fetch_campaign_performance", + "arguments": { + "campaign_id": "", + "metrics": "[\"impressions\", \"clicks\", \"conversions\", \"cost_per_conversion\"]" + } + } + ], + "scores": [ + { + "parse": true, + "name_ok": true, + "schema_ok": true, + "expected_keys": [ + "budget", + "campaign_name", + "end_date", + "platforms", + "start_date", + "target_audience" + ], + "given_keys": [ + "budget", + "campaign_name", + "end_date", + "platforms", + "start_date", + "target_audience" + ] + }, + { + "parse": true, + "name_ok": true, + "schema_ok": true, + "expected_keys": [ + "campaign_id", + "metrics" + ], + "given_keys": [ + "campaign_id", + "metrics" + ] + } + ] + }, + { + "prompt": "I am planning an interactive virtual game night named \"FunTriviaNight\" and requi", + "reply_first_160": "\n\n\nFunTriviaNight\n\n\n10\n\n bool:`", + "ast_ok": false, + "compile_ok": null, + "truncated": 0, + "reply_first_160": "```python\\nimport re\\ndef is_palindrome(s: str) -> bool:\\n # Remove non-alphanumeric characters and convert to lowercase\\n cleaned = re.sub(r'[^a-zA-Z0-9]" + }, + { + "prompt": "Write a Python function `def merge_intervals(intervals: list", + "ast_ok": false, + "compile_ok": null, + "truncated": 0, + "reply_first_160": "```python\\nfrom typing import List\\n\\ndef merge_intervals(intervals: List[List[int]]) -> List[List[int]]:\\n if not intervals:\\n return []\\n \\n #" + }, + { + "prompt": "Write a Python function `def two_sum(nums: list[int], target", + "ast_ok": false, + "compile_ok": null, + "truncated": 0, + "reply_first_160": "```python\\nfrom typing import List\\n\\ndef two_sum(nums: List[int], target: int) -> List[int]:\\n num_to_index = {}\\n for i, num in enumerate(nums):\\n " + }, + { + "prompt": "Write a Python function `def flatten(nested: list) -> list:`", + "ast_ok": false, + "compile_ok": null, + "truncated": 0, + "reply_first_160": "```python\\nfrom collections import deque\\n\\ndef flatten(nested):\\n result = []\\n stack = deque([iter(nested)])\\n while stack:\\n try:\\n " + }, + { + "prompt": "Write a Python function `def parse_csv_line(line: str) -> li", + "ast_ok": false, + "compile_ok": null, + "truncated": 0, + "reply_first_160": "```python\\nimport re\\n\\ndef parse_csv_line(line: str) -> list[str]:\\n # Regular expression to match CSV fields, handling quoted fields and escaped quotes\\n " + }, + { + "prompt": "Write a Python function `def lru_cache(k: int):` returning a", + "ast_ok": false, + "compile_ok": null, + "truncated": 0, + "reply_first_160": "```python\\nfrom collections import OrderedDict\\ndef lru_cache(k):\\n cache = OrderedDict()\\n def decorator(func):\\n def wrapper(*args, **kwargs):\\n " + }, + { + "prompt": "Write a Python function `def is_anagram(a: str, b: str) -> b", + "ast_ok": false, + "compile_ok": null, + "truncated": 0, + "reply_first_160": "```python\\nfrom collections import Counter\\ndef is_anagram(a: str, b: str) -> bool:\\n # Remove non-alphabetic characters and convert to lowercase\\n clean_" + }, + { + "prompt": "Write a Python function `def topological_order(graph: dict[s", + "ast_ok": false, + "compile_ok": null, + "truncated": 0, + "reply_first_160": "```python\\nfrom collections import deque\\n\\ndef topological_order(graph):\\n # Calculate in-degrees for each node\\n in_degree = {node: 0 for node in graph}" + }, + { + "prompt": "Write a Python function `def tokenise(s: str) -> list[str]:`", + "ast_ok": false, + "compile_ok": null, + "truncated": 0, + "reply_first_160": "```python\\nfrom typing import List\\n\\ndef tokenise(s: str) -> List[str]:\\n tokens = []\\n i = 0\\n while i < len(s):\\n if s[i].isdigit():\\n " + }, + { + "prompt": "Write a Python function `def slugify(text: str) -> str:` pro", + "ast_ok": false, + "compile_ok": null, + "truncated": 0, + "reply_first_160": "```python\\nfrom unicodedata import normalize\\n\\ndef slugify(text: str) -> str:\\n # Normalize the text to NFKD form (decomposes characters into base character" + }, + { + "prompt": "Write a Python function `def read_jsonl(path: str) -> list[d", + "ast_ok": false, + "compile_ok": null, + "truncated": 0, + "reply_first_160": "```python\\nimport json\\n\\ndef read_jsonl(path: str) -> list[dict]:\\n with open(path, 'r') as f:\\n for line in f:\\n yield json.loads(line)\\n" + }, + { + "prompt": "Write a Python function `def binary_search(arr: list[int], t", + "ast_ok": false, + "compile_ok": null, + "truncated": 0, + "reply_first_160": "```python\\nfrom typing import List\\n\\ndef binary_search(arr: List[int], target: int) -> int:\\n left, right = 0, len(arr) - 1\\n while left <= right:\\n " + }, + { + "prompt": "Write a Python function `def unique_in_order(s: str) -> list", + "ast_ok": false, + "compile_ok": null, + "truncated": 0, + "reply_first_160": "```python\\nfrom typing import List\\n\\ndef unique_in_order(s: str) -> List[str]:\\n if not s:\\n return []\\n result = [s[0]]\\n for i in range(1, le" + }, + { + "prompt": "Write a Python function `def safe_eval(expr: str) -> int:` e", + "ast_ok": false, + "compile_ok": null, + "truncated": 0, + "reply_first_160": "```python\\ndef safe_eval(expr: str) -> int:\\n # Remove spaces\\n expr = expr.replace(' ', '')\\n \\n # Check for invalid characters\\n valid_chars = " + }, + { + "prompt": "Write a Python function `def dedupe_preserve_order(items: li", + "ast_ok": false, + "compile_ok": null, + "truncated": 0, + "reply_first_160": "```python\\nfrom collections import OrderedDict\\ndef dedupe_preserve_order(items):\\n return list(OrderedDict.fromkeys(items))\\n```\\n" + } + ] + }, + { + "section": "security_mcq", + "n_questions": 0, + "accuracy": 0.0, + "details": [] + } + ] +} \ No newline at end of file