File size: 1,266 Bytes
64a6fc8
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
{
  "benchmark": "verified-trust-pass",
  "date": "2026-07-25",
  "verifier": "SakThai Agent",
  "engine": "llama.cpp Q4_K_M, 2 cores, 2 threads, temp=0.1",
  "methodology": "Each test run 1x with timeout=20s. Same prompt format across all models. Results verified by SakThai.",
  "results": {
    "tool_calling": {
      "1.5B": {"get_weather": "pass", "search_web": "pass", "calculate": "pass", "get_time": "pass", "irrelevance": "pass", "score": "5/5", "verified": true},
      "0.5B": {"get_weather": "pass", "search_web": "pass", "calculate": "pass", "get_time": "pass", "irrelevance": "fail", "score": "4/5", "verified": true}
    },
    "coding": {
      "Coder 1.5B": {"factorial": "pass", "debugging": "pass", "async_explain": "pass", "refactor": "pass", "primes": "pass", "score": "5/5", "verified": true}
    },
    "speed": {
      "0.5B": {"size": "380 MB", "time": "2s", "speed": "24 tok/s", "verified": true},
      "1.5B": {"size": "934 MB", "time": "5s", "speed": "9 tok/s", "verified": true},
      "Coder": {"size": "1.1 GB", "time": "6s", "speed": "9 tok/s", "verified": true}
    }
  },
  "previous_results": {
    "1.5B_bfcl_v1": "4/5",
    "1.5B_bfcl_v2_verified": "5/5",
    "improvement": "+1 score (now handles search_web correctly)"
  }
}