Copilot Copilot commited on
Commit
d67f090
·
1 Parent(s): b3a33d7

fix(models): drop GitHub Llama-3.1 (no tool-calling via Azure endpoint)

Browse files

Audit (scripts/test-tools.mjs) of all 15 configured models against the
calculator tool found:
- 9 OK: gemini-2.5-flash, all 6 Groq, gpt-4o-mini, gpt-4o
- 4 quota-exhausted today (Gemini free-tier, retry tomorrow)
- 2 broken: github Meta-Llama-3.1-8B/405B-Instruct never emit tool-call,
they only output prose 'tôi sẽ dùng calculator', so agent flows fail.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

Files changed (2) hide show
  1. lib/providers.ts +3 -3
  2. scripts/test-tools.mjs +149 -0
lib/providers.ts CHANGED
@@ -32,12 +32,12 @@ export const PROVIDERS = {
32
  label: "GitHub Models",
33
  envKey: "GITHUB_TOKEN",
34
  // Verified active 2026-05-20 against https://models.inference.ai.azure.com/models.
35
- // Phi/Mistral/Cohere are no longer listed for chat-completion via this endpoint.
 
 
36
  models: [
37
  "gpt-4o-mini",
38
  "gpt-4o",
39
- "Meta-Llama-3.1-8B-Instruct",
40
- "Meta-Llama-3.1-405B-Instruct",
41
  ],
42
  default: "gpt-4o-mini",
43
  },
 
32
  label: "GitHub Models",
33
  envKey: "GITHUB_TOKEN",
34
  // Verified active 2026-05-20 against https://models.inference.ai.azure.com/models.
35
+ // Llama-3.1-8B / 405B are listed but DO NOT support function-calling through the
36
+ // Azure inference endpoint (they reply in prose without emitting tool-calls), so
37
+ // they are unusable for an agent UI. Removed 2026-05-20.
38
  models: [
39
  "gpt-4o-mini",
40
  "gpt-4o",
 
 
41
  ],
42
  default: "gpt-4o-mini",
43
  },
scripts/test-tools.mjs ADDED
@@ -0,0 +1,149 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ // Test tool-calling support across all configured models.
2
+ import { readFileSync } from "node:fs"
3
+ import { streamText, tool } from "ai"
4
+ import { createGoogleGenerativeAI } from "@ai-sdk/google"
5
+ import { createGroq } from "@ai-sdk/groq"
6
+ import { createOpenAI } from "@ai-sdk/openai"
7
+ import { z } from "zod"
8
+
9
+ // Tiny inline .env.local loader (avoids extra dep)
10
+ try {
11
+ const env = readFileSync(".env.local", "utf8")
12
+ for (const line of env.split("\n")) {
13
+ const m = line.match(/^\s*([A-Z_][A-Z0-9_]*)\s*=\s*"?([^"\r\n]*)"?\s*$/)
14
+ if (m && !process.env[m[1]]) process.env[m[1]] = m[2]
15
+ }
16
+ } catch {}
17
+
18
+ const PROVIDERS = {
19
+ google: {
20
+ models: [
21
+ "gemini-2.5-flash-lite",
22
+ "gemini-2.5-flash",
23
+ "gemini-2.5-pro",
24
+ "gemini-2.0-flash",
25
+ "gemini-2.0-flash-lite",
26
+ ],
27
+ },
28
+ groq: {
29
+ models: [
30
+ "llama-3.3-70b-versatile",
31
+ "llama-3.1-8b-instant",
32
+ "openai/gpt-oss-120b",
33
+ "openai/gpt-oss-20b",
34
+ "qwen/qwen3-32b",
35
+ "meta-llama/llama-4-scout-17b-16e-instruct",
36
+ ],
37
+ },
38
+ github: {
39
+ models: [
40
+ "gpt-4o-mini",
41
+ "gpt-4o",
42
+ "Meta-Llama-3.1-8B-Instruct",
43
+ "Meta-Llama-3.1-405B-Instruct",
44
+ ],
45
+ },
46
+ }
47
+
48
+ const tools = {
49
+ calculator: tool({
50
+ description: "Evaluate a math expression. Use for arithmetic.",
51
+ parameters: z.object({ expression: z.string() }),
52
+ execute: async ({ expression }) => {
53
+ const r = Function(`"use strict";return (${expression})`)()
54
+ return { result: r }
55
+ },
56
+ }),
57
+ currentTime: tool({
58
+ description: "Get current ISO date-time.",
59
+ parameters: z.object({}),
60
+ execute: async () => ({ now: new Date().toISOString() }),
61
+ }),
62
+ }
63
+
64
+ function build(provider, modelId) {
65
+ if (provider === "google")
66
+ return createGoogleGenerativeAI({ apiKey: process.env.GOOGLE_GENERATIVE_AI_API_KEY })(modelId)
67
+ if (provider === "groq") return createGroq({ apiKey: process.env.GROQ_API_KEY })(modelId)
68
+ if (provider === "github")
69
+ return createOpenAI({
70
+ apiKey: process.env.GITHUB_TOKEN,
71
+ baseURL: "https://models.inference.ai.azure.com",
72
+ })(modelId)
73
+ }
74
+
75
+ async function testOne(provider, modelId) {
76
+ const t0 = Date.now()
77
+ try {
78
+ const result = streamText({
79
+ model: build(provider, modelId),
80
+ tools,
81
+ maxSteps: 3,
82
+ system:
83
+ "Khi user hỏi tính toán PHẢI gọi tool calculator. " +
84
+ "Khi user hỏi giờ PHẢI gọi tool currentTime. Không tự tính.",
85
+ messages: [{ role: "user", content: "Tính giúp: 137 * 29 + 8 = ?" }],
86
+ })
87
+
88
+ let toolCalled = false
89
+ let toolName = null
90
+ let text = ""
91
+ let chunks = 0
92
+ let firstChunkMs = null
93
+
94
+ for await (const part of result.fullStream) {
95
+ chunks++
96
+ if (firstChunkMs === null) firstChunkMs = Date.now() - t0
97
+ if (part.type === "tool-call") {
98
+ toolCalled = true
99
+ toolName = part.toolName
100
+ } else if (part.type === "text-delta") {
101
+ text += part.textDelta
102
+ } else if (part.type === "error") {
103
+ throw part.error
104
+ }
105
+ }
106
+ return {
107
+ ok: true,
108
+ toolCalled,
109
+ toolName,
110
+ streaming: chunks > 3,
111
+ chunks,
112
+ ttfbMs: firstChunkMs,
113
+ totalMs: Date.now() - t0,
114
+ textPreview: text.slice(0, 80).replace(/\s+/g, " "),
115
+ }
116
+ } catch (e) {
117
+ return { ok: false, error: (e?.message || String(e)).slice(0, 240), totalMs: Date.now() - t0 }
118
+ }
119
+ }
120
+
121
+ const results = []
122
+ for (const [provider, info] of Object.entries(PROVIDERS)) {
123
+ for (const m of info.models) {
124
+ process.stdout.write(`Testing ${provider}/${m} ... `)
125
+ const r = await testOne(provider, m)
126
+ const status = r.ok
127
+ ? r.toolCalled
128
+ ? `OK tool=${r.toolName} stream=${r.streaming ? "y" : "n"} chunks=${r.chunks} ttfb=${r.ttfbMs}ms total=${r.totalMs}ms`
129
+ : `NO-TOOL stream=${r.streaming ? "y" : "n"} total=${r.totalMs}ms text="${r.textPreview}"`
130
+ : `ERR ${r.error}`
131
+ console.log(status)
132
+ results.push({ provider, model: m, ...r })
133
+ await new Promise((r) => setTimeout(r, 800))
134
+ }
135
+ }
136
+
137
+ console.log("\n=== SUMMARY ===")
138
+ for (const r of results) {
139
+ const tool = r.ok ? (r.toolCalled ? "OK " : "NO ") : "ERR"
140
+ const stream = r.ok ? (r.streaming ? "y" : "n") : "-"
141
+ console.log(`${tool} stream=${stream} ${r.provider.padEnd(8)} ${r.model}`)
142
+ }
143
+ const broken = results.filter((r) => !r.ok)
144
+ const noTool = results.filter((r) => r.ok && !r.toolCalled)
145
+ console.log(`\n${results.length} tested · ${broken.length} broken · ${noTool.length} no tool-call`)
146
+ if (broken.length) {
147
+ console.log("\nBROKEN:")
148
+ for (const b of broken) console.log(` ${b.provider}/${b.model}: ${b.error}`)
149
+ }