| { |
| "model": "ngari-tool:stable", |
| "dataset": "/home/garry/NGARI/training/eval_tool.json", |
| "num_examples": 20, |
| "total_examples": 20, |
| "tool_detection_rate": 1.0, |
| "tool_name_accuracy": 1.0, |
| "params_validity_rate": 1.0, |
| "tool_detected": 20, |
| "name_correct": 20, |
| "params_valid": 20, |
| "avg_latency_sec": 3.17, |
| "total_time_sec": 63.4, |
| "results": [ |
| { |
| "index": 0, |
| "user_preview": "What's the weather like in Tokyo right now?", |
| "expected_tool": "get_weather", |
| "tool_detected": true, |
| "name_correct": true, |
| "params_valid": true, |
| "actual_tool": "get_weather", |
| "response_preview": "{\n \"name\": \"get_weather\",\n \"arguments\": {\n \"city\": \"Tokyo\"\n }\n}", |
| "status": "CORRECT (get_weather)", |
| "elapsed": 2.4 |
| }, |
| { |
| "index": 1, |
| "user_preview": "Search my knowledge base for information about AI sovereignty.", |
| "expected_tool": "rag_search", |
| "tool_detected": true, |
| "name_correct": true, |
| "params_valid": true, |
| "actual_tool": "rag_search", |
| "response_preview": "{\n \"name\": \"rag_search\",\n \"arguments\": {\n \"query\": \"AI sovereignty\",\n \"top_k\": 5\n }\n}", |
| "status": "CORRECT (rag_search)", |
| "elapsed": 3.12 |
| }, |
| { |
| "index": 2, |
| "user_preview": "Read the NGARi constitution document for me.", |
| "expected_tool": "read_file", |
| "tool_detected": true, |
| "name_correct": true, |
| "params_valid": true, |
| "actual_tool": "read_file", |
| "response_preview": "{\n \"name\": \"read_file\",\n \"arguments\": {\n \"path\": \"/path/to/ngari/constitution/document\"\n }\n}", |
| "status": "CORRECT (read_file)", |
| "elapsed": 2.88 |
| }, |
| { |
| "index": 3, |
| "user_preview": "Set a timer for 15 minutes for my break.", |
| "expected_tool": "set_timer", |
| "tool_detected": true, |
| "name_correct": true, |
| "params_valid": true, |
| "actual_tool": "set_timer", |
| "response_preview": "```json\n{\n \"name\": \"set_timer\",\n \"arguments\": {\n \"duration_minutes\": 15,\n \"label\": \"break\"\n }\n}\n```", |
| "status": "CORRECT (set_timer)", |
| "elapsed": 3.43 |
| }, |
| { |
| "index": 4, |
| "user_preview": "What files are in the directory /home/garry/NGARI?", |
| "expected_tool": "list_directory", |
| "tool_detected": true, |
| "name_correct": true, |
| "params_valid": true, |
| "actual_tool": "list_directory", |
| "response_preview": "{\n \"name\": \"list_directory\",\n \"arguments\": {\n \"path\": \"/home/garry/NGARI\"\n }\n}", |
| "status": "CORRECT (list_directory)", |
| "elapsed": 2.78 |
| }, |
| { |
| "index": 5, |
| "user_preview": "Run a bash command to check disk usage on the root partition.", |
| "expected_tool": "run_shell", |
| "tool_detected": true, |
| "name_correct": true, |
| "params_valid": true, |
| "actual_tool": "run_shell", |
| "response_preview": "```json\n{\n \"name\": \"run_shell\",\n \"arguments\": {\n \"command\": \"df -h /\"\n }\n}\n```", |
| "status": "CORRECT (run_shell)", |
| "elapsed": 2.92 |
| }, |
| { |
| "index": 6, |
| "user_preview": "Check my calendar for tomorrow's schedule.", |
| "expected_tool": "get_calendar", |
| "tool_detected": true, |
| "name_correct": true, |
| "params_valid": true, |
| "actual_tool": "get_calendar", |
| "response_preview": "{\n \"name\": \"get_calendar\",\n \"arguments\": {\n \"date\": \"tomorrow\"\n }\n}", |
| "status": "CORRECT (get_calendar)", |
| "elapsed": 2.47 |
| }, |
| { |
| "index": 7, |
| "user_preview": "Send an email to alice@example.com saying I'll be late.", |
| "expected_tool": "send_email", |
| "tool_detected": true, |
| "name_correct": true, |
| "params_valid": true, |
| "actual_tool": "send_email", |
| "response_preview": "```json\n{\n \"name\": \"send_email\",\n \"arguments\": {\n \"to\": \"alice@example.com\",\n \"subject\": \"I'll be late\",\n \"bo", |
| "status": "CORRECT (send_email)", |
| "elapsed": 6.33 |
| }, |
| { |
| "index": 8, |
| "user_preview": "Find documents related to zero telemetry in the knowledge base.", |
| "expected_tool": "rag_search", |
| "tool_detected": true, |
| "name_correct": true, |
| "params_valid": true, |
| "actual_tool": "rag_search", |
| "response_preview": "{\n \"name\": \"rag_search\",\n \"arguments\": {\n \"query\": \"zero telemetry\",\n \"top_k\": 5\n }\n}", |
| "status": "CORRECT (rag_search)", |
| "elapsed": 3.1 |
| }, |
| { |
| "index": 9, |
| "user_preview": "What is the weather forecast for London this weekend?", |
| "expected_tool": "get_weather", |
| "tool_detected": true, |
| "name_correct": true, |
| "params_valid": true, |
| "actual_tool": "get_weather", |
| "response_preview": "{\n \"name\": \"get_weather\",\n \"arguments\": {\n \"city\": \"London\"\n }\n}", |
| "status": "CORRECT (get_weather)", |
| "elapsed": 2.37 |
| }, |
| { |
| "index": 10, |
| "user_preview": "Read the file /etc/hostname for me.", |
| "expected_tool": "read_file", |
| "tool_detected": true, |
| "name_correct": true, |
| "params_valid": true, |
| "actual_tool": "read_file", |
| "response_preview": "```json\n{\n \"name\": \"read_file\",\n \"arguments\": {\n \"path\": \"/etc/hostname\"\n }\n}\n```", |
| "status": "CORRECT (read_file)", |
| "elapsed": 2.87 |
| }, |
| { |
| "index": 11, |
| "user_preview": "List files in /mnt/ollama-models/ngari-workspace/training/", |
| "expected_tool": "list_directory", |
| "tool_detected": true, |
| "name_correct": true, |
| "params_valid": true, |
| "actual_tool": "list_directory", |
| "response_preview": "```json\n{\n \"name\": \"list_directory\",\n \"arguments\": {\n \"path\": \"/mnt/ollama-models/ngari-workspace/training/\"\n }\n}\n", |
| "status": "CORRECT (list_directory)", |
| "elapsed": 3.58 |
| }, |
| { |
| "index": 12, |
| "user_preview": "Set a timer for 5 minutes for my pasta.", |
| "expected_tool": "set_timer", |
| "tool_detected": true, |
| "name_correct": true, |
| "params_valid": true, |
| "actual_tool": "set_timer", |
| "response_preview": "{\n \"name\": \"set_timer\",\n \"arguments\": {\n \"duration_minutes\": 5,\n \"label\": \"pasta\"\n }\n}", |
| "status": "CORRECT (set_timer)", |
| "elapsed": 3.06 |
| }, |
| { |
| "index": 13, |
| "user_preview": "What is the temperature in New York right now?", |
| "expected_tool": "get_weather", |
| "tool_detected": true, |
| "name_correct": true, |
| "params_valid": true, |
| "actual_tool": "get_weather", |
| "response_preview": "{\n \"name\": \"get_weather\",\n \"arguments\": {\n \"city\": \"New York\"\n }\n}", |
| "status": "CORRECT (get_weather)", |
| "elapsed": 2.43 |
| }, |
| { |
| "index": 14, |
| "user_preview": "Run df -h and show me the output.", |
| "expected_tool": "run_shell", |
| "tool_detected": true, |
| "name_correct": true, |
| "params_valid": true, |
| "actual_tool": "run_shell", |
| "response_preview": "```json\n{\n \"name\": \"run_shell\",\n \"arguments\": {\n \"command\": \"df -h\"\n }\n}\n```", |
| "status": "CORRECT (run_shell)", |
| "elapsed": 2.9 |
| }, |
| { |
| "index": 15, |
| "user_preview": "Send a message to bob@corp.com about the project deadline.", |
| "expected_tool": "send_email", |
| "tool_detected": true, |
| "name_correct": true, |
| "params_valid": true, |
| "actual_tool": "send_email", |
| "response_preview": "{\n \"name\": \"send_email\",\n \"arguments\": {\n \"to\": \"bob@corp.com\",\n \"subject\": \"Project Deadline Reminder\",\n \"bo", |
| "status": "CORRECT (send_email)", |
| "elapsed": 6.02 |
| }, |
| { |
| "index": 16, |
| "user_preview": "What's the weather like in Paris?", |
| "expected_tool": "get_weather", |
| "tool_detected": true, |
| "name_correct": true, |
| "params_valid": true, |
| "actual_tool": "get_weather", |
| "response_preview": "{\n \"name\": \"get_weather\",\n \"arguments\": {\n \"city\": \"Paris\"\n }\n}", |
| "status": "CORRECT (get_weather)", |
| "elapsed": 2.38 |
| }, |
| { |
| "index": 17, |
| "user_preview": "Search for 'audit by construction' in my documents.", |
| "expected_tool": "rag_search", |
| "tool_detected": true, |
| "name_correct": true, |
| "params_valid": true, |
| "actual_tool": "rag_search", |
| "response_preview": "```json\n{\n \"name\": \"rag_search\",\n \"arguments\": {\n \"query\": \"audit by construction\",\n \"top_k\": 5\n }\n}\n```", |
| "status": "CORRECT (rag_search)", |
| "elapsed": 3.54 |
| }, |
| { |
| "index": 18, |
| "user_preview": "Check what's on my calendar for today.", |
| "expected_tool": "get_calendar", |
| "tool_detected": true, |
| "name_correct": true, |
| "params_valid": true, |
| "actual_tool": "get_calendar", |
| "response_preview": "{\n \"name\": \"get_calendar\",\n \"arguments\": {\n \"date\": \"today\"\n }\n}", |
| "status": "CORRECT (get_calendar)", |
| "elapsed": 2.37 |
| }, |
| { |
| "index": 19, |
| "user_preview": "Read the README.md file in my project root.", |
| "expected_tool": "read_file", |
| "tool_detected": true, |
| "name_correct": true, |
| "params_valid": true, |
| "actual_tool": "read_file", |
| "response_preview": "{\n \"name\": \"read_file\",\n \"arguments\": {\n \"path\": \"./README.md\"\n }\n}", |
| "status": "CORRECT (read_file)", |
| "elapsed": 2.45 |
| } |
| ] |
| } |
