Hugging Face's logo

NGARiAI
/
ngari-tool

Safetensors
GGUF
English
qwen2
ngari
sovereign-ai
edge-ai
qwen2.5
lora
conversational
Model card Files
xet
 Community
garryjohnsoniii commited on
Commit
02a000c
·
verified ·
1 Parent(s): ba5456d

Upload folder using huggingface_hub

Browse files
Files changed (1) hide show
  1. ngari-tool-stable_tool_eval.json +256 -0
ngari-tool-stable_tool_eval.json ADDED
@@ -0,0 +1,256 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "model": "ngari-tool:stable",
3
+ "dataset": "/home/garry/NGARI/training/eval_tool.json",
4
+ "num_examples": 20,
5
+ "total_examples": 20,
6
+ "tool_detection_rate": 1.0,
7
+ "tool_name_accuracy": 1.0,
8
+ "params_validity_rate": 1.0,
9
+ "tool_detected": 20,
10
+ "name_correct": 20,
11
+ "params_valid": 20,
12
+ "avg_latency_sec": 3.17,
13
+ "total_time_sec": 63.4,
14
+ "results": [
15
+ {
16
+ "index": 0,
17
+ "user_preview": "What's the weather like in Tokyo right now?",
18
+ "expected_tool": "get_weather",
19
+ "tool_detected": true,
20
+ "name_correct": true,
21
+ "params_valid": true,
22
+ "actual_tool": "get_weather",
23
+ "response_preview": "{\n \"name\": \"get_weather\",\n \"arguments\": {\n \"city\": \"Tokyo\"\n }\n}",
24
+ "status": "CORRECT (get_weather)",
25
+ "elapsed": 2.4
26
+ },
27
+ {
28
+ "index": 1,
29
+ "user_preview": "Search my knowledge base for information about AI sovereignty.",
30
+ "expected_tool": "rag_search",
31
+ "tool_detected": true,
32
+ "name_correct": true,
33
+ "params_valid": true,
34
+ "actual_tool": "rag_search",
35
+ "response_preview": "{\n \"name\": \"rag_search\",\n \"arguments\": {\n \"query\": \"AI sovereignty\",\n \"top_k\": 5\n }\n}",
36
+ "status": "CORRECT (rag_search)",
37
+ "elapsed": 3.12
38
+ },
39
+ {
40
+ "index": 2,
41
+ "user_preview": "Read the NGARi constitution document for me.",
42
+ "expected_tool": "read_file",
43
+ "tool_detected": true,
44
+ "name_correct": true,
45
+ "params_valid": true,
46
+ "actual_tool": "read_file",
47
+ "response_preview": "{\n \"name\": \"read_file\",\n \"arguments\": {\n \"path\": \"/path/to/ngari/constitution/document\"\n }\n}",
48
+ "status": "CORRECT (read_file)",
49
+ "elapsed": 2.88
50
+ },
51
+ {
52
+ "index": 3,
53
+ "user_preview": "Set a timer for 15 minutes for my break.",
54
+ "expected_tool": "set_timer",
55
+ "tool_detected": true,
56
+ "name_correct": true,
57
+ "params_valid": true,
58
+ "actual_tool": "set_timer",
59
+ "response_preview": "```json\n{\n \"name\": \"set_timer\",\n \"arguments\": {\n \"duration_minutes\": 15,\n \"label\": \"break\"\n }\n}\n```",
60
+ "status": "CORRECT (set_timer)",
61
+ "elapsed": 3.43
62
+ },
63
+ {
64
+ "index": 4,
65
+ "user_preview": "What files are in the directory /home/garry/NGARI?",
66
+ "expected_tool": "list_directory",
67
+ "tool_detected": true,
68
+ "name_correct": true,
69
+ "params_valid": true,
70
+ "actual_tool": "list_directory",
71
+ "response_preview": "{\n \"name\": \"list_directory\",\n \"arguments\": {\n \"path\": \"/home/garry/NGARI\"\n }\n}",
72
+ "status": "CORRECT (list_directory)",
73
+ "elapsed": 2.78
74
+ },
75
+ {
76
+ "index": 5,
77
+ "user_preview": "Run a bash command to check disk usage on the root partition.",
78
+ "expected_tool": "run_shell",
79
+ "tool_detected": true,
80
+ "name_correct": true,
81
+ "params_valid": true,
82
+ "actual_tool": "run_shell",
83
+ "response_preview": "```json\n{\n \"name\": \"run_shell\",\n \"arguments\": {\n \"command\": \"df -h /\"\n }\n}\n```",
84
+ "status": "CORRECT (run_shell)",
85
+ "elapsed": 2.92
86
+ },
87
+ {
88
+ "index": 6,
89
+ "user_preview": "Check my calendar for tomorrow's schedule.",
90
+ "expected_tool": "get_calendar",
91
+ "tool_detected": true,
92
+ "name_correct": true,
93
+ "params_valid": true,
94
+ "actual_tool": "get_calendar",
95
+ "response_preview": "{\n \"name\": \"get_calendar\",\n \"arguments\": {\n \"date\": \"tomorrow\"\n }\n}",
96
+ "status": "CORRECT (get_calendar)",
97
+ "elapsed": 2.47
98
+ },
99
+ {
100
+ "index": 7,
101
+ "user_preview": "Send an email to alice@example.com saying I'll be late.",
102
+ "expected_tool": "send_email",
103
+ "tool_detected": true,
104
+ "name_correct": true,
105
+ "params_valid": true,
106
+ "actual_tool": "send_email",
107
+ "response_preview": "```json\n{\n \"name\": \"send_email\",\n \"arguments\": {\n \"to\": \"alice@example.com\",\n \"subject\": \"I'll be late\",\n \"bo",
108
+ "status": "CORRECT (send_email)",
109
+ "elapsed": 6.33
110
+ },
111
+ {
112
+ "index": 8,
113
+ "user_preview": "Find documents related to zero telemetry in the knowledge base.",
114
+ "expected_tool": "rag_search",
115
+ "tool_detected": true,
116
+ "name_correct": true,
117
+ "params_valid": true,
118
+ "actual_tool": "rag_search",
119
+ "response_preview": "{\n \"name\": \"rag_search\",\n \"arguments\": {\n \"query\": \"zero telemetry\",\n \"top_k\": 5\n }\n}",
120
+ "status": "CORRECT (rag_search)",
121
+ "elapsed": 3.1
122
+ },
123
+ {
124
+ "index": 9,
125
+ "user_preview": "What is the weather forecast for London this weekend?",
126
+ "expected_tool": "get_weather",
127
+ "tool_detected": true,
128
+ "name_correct": true,
129
+ "params_valid": true,
130
+ "actual_tool": "get_weather",
131
+ "response_preview": "{\n \"name\": \"get_weather\",\n \"arguments\": {\n \"city\": \"London\"\n }\n}",
132
+ "status": "CORRECT (get_weather)",
133
+ "elapsed": 2.37
134
+ },
135
+ {
136
+ "index": 10,
137
+ "user_preview": "Read the file /etc/hostname for me.",
138
+ "expected_tool": "read_file",
139
+ "tool_detected": true,
140
+ "name_correct": true,
141
+ "params_valid": true,
142
+ "actual_tool": "read_file",
143
+ "response_preview": "```json\n{\n \"name\": \"read_file\",\n \"arguments\": {\n \"path\": \"/etc/hostname\"\n }\n}\n```",
144
+ "status": "CORRECT (read_file)",
145
+ "elapsed": 2.87
146
+ },
147
+ {
148
+ "index": 11,
149
+ "user_preview": "List files in /mnt/ollama-models/ngari-workspace/training/",
150
+ "expected_tool": "list_directory",
151
+ "tool_detected": true,
152
+ "name_correct": true,
153
+ "params_valid": true,
154
+ "actual_tool": "list_directory",
155
+ "response_preview": "```json\n{\n \"name\": \"list_directory\",\n \"arguments\": {\n \"path\": \"/mnt/ollama-models/ngari-workspace/training/\"\n }\n}\n",
156
+ "status": "CORRECT (list_directory)",
157
+ "elapsed": 3.58
158
+ },
159
+ {
160
+ "index": 12,
161
+ "user_preview": "Set a timer for 5 minutes for my pasta.",
162
+ "expected_tool": "set_timer",
163
+ "tool_detected": true,
164
+ "name_correct": true,
165
+ "params_valid": true,
166
+ "actual_tool": "set_timer",
167
+ "response_preview": "{\n \"name\": \"set_timer\",\n \"arguments\": {\n \"duration_minutes\": 5,\n \"label\": \"pasta\"\n }\n}",
168
+ "status": "CORRECT (set_timer)",
169
+ "elapsed": 3.06
170
+ },
171
+ {
172
+ "index": 13,
173
+ "user_preview": "What is the temperature in New York right now?",
174
+ "expected_tool": "get_weather",
175
+ "tool_detected": true,
176
+ "name_correct": true,
177
+ "params_valid": true,
178
+ "actual_tool": "get_weather",
179
+ "response_preview": "{\n \"name\": \"get_weather\",\n \"arguments\": {\n \"city\": \"New York\"\n }\n}",
180
+ "status": "CORRECT (get_weather)",
181
+ "elapsed": 2.43
182
+ },
183
+ {
184
+ "index": 14,
185
+ "user_preview": "Run df -h and show me the output.",
186
+ "expected_tool": "run_shell",
187
+ "tool_detected": true,
188
+ "name_correct": true,
189
+ "params_valid": true,
190
+ "actual_tool": "run_shell",
191
+ "response_preview": "```json\n{\n \"name\": \"run_shell\",\n \"arguments\": {\n \"command\": \"df -h\"\n }\n}\n```",
192
+ "status": "CORRECT (run_shell)",
193
+ "elapsed": 2.9
194
+ },
195
+ {
196
+ "index": 15,
197
+ "user_preview": "Send a message to bob@corp.com about the project deadline.",
198
+ "expected_tool": "send_email",
199
+ "tool_detected": true,
200
+ "name_correct": true,
201
+ "params_valid": true,
202
+ "actual_tool": "send_email",
203
+ "response_preview": "{\n \"name\": \"send_email\",\n \"arguments\": {\n \"to\": \"bob@corp.com\",\n \"subject\": \"Project Deadline Reminder\",\n \"bo",
204
+ "status": "CORRECT (send_email)",
205
+ "elapsed": 6.02
206
+ },
207
+ {
208
+ "index": 16,
209
+ "user_preview": "What's the weather like in Paris?",
210
+ "expected_tool": "get_weather",
211
+ "tool_detected": true,
212
+ "name_correct": true,
213
+ "params_valid": true,
214
+ "actual_tool": "get_weather",
215
+ "response_preview": "{\n \"name\": \"get_weather\",\n \"arguments\": {\n \"city\": \"Paris\"\n }\n}",
216
+ "status": "CORRECT (get_weather)",
217
+ "elapsed": 2.38
218
+ },
219
+ {
220
+ "index": 17,
221
+ "user_preview": "Search for 'audit by construction' in my documents.",
222
+ "expected_tool": "rag_search",
223
+ "tool_detected": true,
224
+ "name_correct": true,
225
+ "params_valid": true,
226
+ "actual_tool": "rag_search",
227
+ "response_preview": "```json\n{\n \"name\": \"rag_search\",\n \"arguments\": {\n \"query\": \"audit by construction\",\n \"top_k\": 5\n }\n}\n```",
228
+ "status": "CORRECT (rag_search)",
229
+ "elapsed": 3.54
230
+ },
231
+ {
232
+ "index": 18,
233
+ "user_preview": "Check what's on my calendar for today.",
234
+ "expected_tool": "get_calendar",
235
+ "tool_detected": true,
236
+ "name_correct": true,
237
+ "params_valid": true,
238
+ "actual_tool": "get_calendar",
239
+ "response_preview": "{\n \"name\": \"get_calendar\",\n \"arguments\": {\n \"date\": \"today\"\n }\n}",
240
+ "status": "CORRECT (get_calendar)",
241
+ "elapsed": 2.37
242
+ },
243
+ {
244
+ "index": 19,
245
+ "user_preview": "Read the README.md file in my project root.",
246
+ "expected_tool": "read_file",
247
+ "tool_detected": true,
248
+ "name_correct": true,
249
+ "params_valid": true,
250
+ "actual_tool": "read_file",
251
+ "response_preview": "{\n \"name\": \"read_file\",\n \"arguments\": {\n \"path\": \"./README.md\"\n }\n}",
252
+ "status": "CORRECT (read_file)",
253
+ "elapsed": 2.45
254
+ }
255
+ ]
256
+ }