Spaces:
Runtime error
Runtime error
Copilot Copilot commited on
Commit ·
d67f090
1
Parent(s): b3a33d7
fix(models): drop GitHub Llama-3.1 (no tool-calling via Azure endpoint)
Browse filesAudit (scripts/test-tools.mjs) of all 15 configured models against the
calculator tool found:
- 9 OK: gemini-2.5-flash, all 6 Groq, gpt-4o-mini, gpt-4o
- 4 quota-exhausted today (Gemini free-tier, retry tomorrow)
- 2 broken: github Meta-Llama-3.1-8B/405B-Instruct never emit tool-call,
they only output prose 'tôi sẽ dùng calculator', so agent flows fail.
Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
- lib/providers.ts +3 -3
- scripts/test-tools.mjs +149 -0
lib/providers.ts
CHANGED
|
@@ -32,12 +32,12 @@ export const PROVIDERS = {
|
|
| 32 |
label: "GitHub Models",
|
| 33 |
envKey: "GITHUB_TOKEN",
|
| 34 |
// Verified active 2026-05-20 against https://models.inference.ai.azure.com/models.
|
| 35 |
-
//
|
|
|
|
|
|
|
| 36 |
models: [
|
| 37 |
"gpt-4o-mini",
|
| 38 |
"gpt-4o",
|
| 39 |
-
"Meta-Llama-3.1-8B-Instruct",
|
| 40 |
-
"Meta-Llama-3.1-405B-Instruct",
|
| 41 |
],
|
| 42 |
default: "gpt-4o-mini",
|
| 43 |
},
|
|
|
|
| 32 |
label: "GitHub Models",
|
| 33 |
envKey: "GITHUB_TOKEN",
|
| 34 |
// Verified active 2026-05-20 against https://models.inference.ai.azure.com/models.
|
| 35 |
+
// Llama-3.1-8B / 405B are listed but DO NOT support function-calling through the
|
| 36 |
+
// Azure inference endpoint (they reply in prose without emitting tool-calls), so
|
| 37 |
+
// they are unusable for an agent UI. Removed 2026-05-20.
|
| 38 |
models: [
|
| 39 |
"gpt-4o-mini",
|
| 40 |
"gpt-4o",
|
|
|
|
|
|
|
| 41 |
],
|
| 42 |
default: "gpt-4o-mini",
|
| 43 |
},
|
scripts/test-tools.mjs
ADDED
|
@@ -0,0 +1,149 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// Test tool-calling support across all configured models.
|
| 2 |
+
import { readFileSync } from "node:fs"
|
| 3 |
+
import { streamText, tool } from "ai"
|
| 4 |
+
import { createGoogleGenerativeAI } from "@ai-sdk/google"
|
| 5 |
+
import { createGroq } from "@ai-sdk/groq"
|
| 6 |
+
import { createOpenAI } from "@ai-sdk/openai"
|
| 7 |
+
import { z } from "zod"
|
| 8 |
+
|
| 9 |
+
// Tiny inline .env.local loader (avoids extra dep)
|
| 10 |
+
try {
|
| 11 |
+
const env = readFileSync(".env.local", "utf8")
|
| 12 |
+
for (const line of env.split("\n")) {
|
| 13 |
+
const m = line.match(/^\s*([A-Z_][A-Z0-9_]*)\s*=\s*"?([^"\r\n]*)"?\s*$/)
|
| 14 |
+
if (m && !process.env[m[1]]) process.env[m[1]] = m[2]
|
| 15 |
+
}
|
| 16 |
+
} catch {}
|
| 17 |
+
|
| 18 |
+
const PROVIDERS = {
|
| 19 |
+
google: {
|
| 20 |
+
models: [
|
| 21 |
+
"gemini-2.5-flash-lite",
|
| 22 |
+
"gemini-2.5-flash",
|
| 23 |
+
"gemini-2.5-pro",
|
| 24 |
+
"gemini-2.0-flash",
|
| 25 |
+
"gemini-2.0-flash-lite",
|
| 26 |
+
],
|
| 27 |
+
},
|
| 28 |
+
groq: {
|
| 29 |
+
models: [
|
| 30 |
+
"llama-3.3-70b-versatile",
|
| 31 |
+
"llama-3.1-8b-instant",
|
| 32 |
+
"openai/gpt-oss-120b",
|
| 33 |
+
"openai/gpt-oss-20b",
|
| 34 |
+
"qwen/qwen3-32b",
|
| 35 |
+
"meta-llama/llama-4-scout-17b-16e-instruct",
|
| 36 |
+
],
|
| 37 |
+
},
|
| 38 |
+
github: {
|
| 39 |
+
models: [
|
| 40 |
+
"gpt-4o-mini",
|
| 41 |
+
"gpt-4o",
|
| 42 |
+
"Meta-Llama-3.1-8B-Instruct",
|
| 43 |
+
"Meta-Llama-3.1-405B-Instruct",
|
| 44 |
+
],
|
| 45 |
+
},
|
| 46 |
+
}
|
| 47 |
+
|
| 48 |
+
const tools = {
|
| 49 |
+
calculator: tool({
|
| 50 |
+
description: "Evaluate a math expression. Use for arithmetic.",
|
| 51 |
+
parameters: z.object({ expression: z.string() }),
|
| 52 |
+
execute: async ({ expression }) => {
|
| 53 |
+
const r = Function(`"use strict";return (${expression})`)()
|
| 54 |
+
return { result: r }
|
| 55 |
+
},
|
| 56 |
+
}),
|
| 57 |
+
currentTime: tool({
|
| 58 |
+
description: "Get current ISO date-time.",
|
| 59 |
+
parameters: z.object({}),
|
| 60 |
+
execute: async () => ({ now: new Date().toISOString() }),
|
| 61 |
+
}),
|
| 62 |
+
}
|
| 63 |
+
|
| 64 |
+
function build(provider, modelId) {
|
| 65 |
+
if (provider === "google")
|
| 66 |
+
return createGoogleGenerativeAI({ apiKey: process.env.GOOGLE_GENERATIVE_AI_API_KEY })(modelId)
|
| 67 |
+
if (provider === "groq") return createGroq({ apiKey: process.env.GROQ_API_KEY })(modelId)
|
| 68 |
+
if (provider === "github")
|
| 69 |
+
return createOpenAI({
|
| 70 |
+
apiKey: process.env.GITHUB_TOKEN,
|
| 71 |
+
baseURL: "https://models.inference.ai.azure.com",
|
| 72 |
+
})(modelId)
|
| 73 |
+
}
|
| 74 |
+
|
| 75 |
+
async function testOne(provider, modelId) {
|
| 76 |
+
const t0 = Date.now()
|
| 77 |
+
try {
|
| 78 |
+
const result = streamText({
|
| 79 |
+
model: build(provider, modelId),
|
| 80 |
+
tools,
|
| 81 |
+
maxSteps: 3,
|
| 82 |
+
system:
|
| 83 |
+
"Khi user hỏi tính toán PHẢI gọi tool calculator. " +
|
| 84 |
+
"Khi user hỏi giờ PHẢI gọi tool currentTime. Không tự tính.",
|
| 85 |
+
messages: [{ role: "user", content: "Tính giúp: 137 * 29 + 8 = ?" }],
|
| 86 |
+
})
|
| 87 |
+
|
| 88 |
+
let toolCalled = false
|
| 89 |
+
let toolName = null
|
| 90 |
+
let text = ""
|
| 91 |
+
let chunks = 0
|
| 92 |
+
let firstChunkMs = null
|
| 93 |
+
|
| 94 |
+
for await (const part of result.fullStream) {
|
| 95 |
+
chunks++
|
| 96 |
+
if (firstChunkMs === null) firstChunkMs = Date.now() - t0
|
| 97 |
+
if (part.type === "tool-call") {
|
| 98 |
+
toolCalled = true
|
| 99 |
+
toolName = part.toolName
|
| 100 |
+
} else if (part.type === "text-delta") {
|
| 101 |
+
text += part.textDelta
|
| 102 |
+
} else if (part.type === "error") {
|
| 103 |
+
throw part.error
|
| 104 |
+
}
|
| 105 |
+
}
|
| 106 |
+
return {
|
| 107 |
+
ok: true,
|
| 108 |
+
toolCalled,
|
| 109 |
+
toolName,
|
| 110 |
+
streaming: chunks > 3,
|
| 111 |
+
chunks,
|
| 112 |
+
ttfbMs: firstChunkMs,
|
| 113 |
+
totalMs: Date.now() - t0,
|
| 114 |
+
textPreview: text.slice(0, 80).replace(/\s+/g, " "),
|
| 115 |
+
}
|
| 116 |
+
} catch (e) {
|
| 117 |
+
return { ok: false, error: (e?.message || String(e)).slice(0, 240), totalMs: Date.now() - t0 }
|
| 118 |
+
}
|
| 119 |
+
}
|
| 120 |
+
|
| 121 |
+
const results = []
|
| 122 |
+
for (const [provider, info] of Object.entries(PROVIDERS)) {
|
| 123 |
+
for (const m of info.models) {
|
| 124 |
+
process.stdout.write(`Testing ${provider}/${m} ... `)
|
| 125 |
+
const r = await testOne(provider, m)
|
| 126 |
+
const status = r.ok
|
| 127 |
+
? r.toolCalled
|
| 128 |
+
? `OK tool=${r.toolName} stream=${r.streaming ? "y" : "n"} chunks=${r.chunks} ttfb=${r.ttfbMs}ms total=${r.totalMs}ms`
|
| 129 |
+
: `NO-TOOL stream=${r.streaming ? "y" : "n"} total=${r.totalMs}ms text="${r.textPreview}"`
|
| 130 |
+
: `ERR ${r.error}`
|
| 131 |
+
console.log(status)
|
| 132 |
+
results.push({ provider, model: m, ...r })
|
| 133 |
+
await new Promise((r) => setTimeout(r, 800))
|
| 134 |
+
}
|
| 135 |
+
}
|
| 136 |
+
|
| 137 |
+
console.log("\n=== SUMMARY ===")
|
| 138 |
+
for (const r of results) {
|
| 139 |
+
const tool = r.ok ? (r.toolCalled ? "OK " : "NO ") : "ERR"
|
| 140 |
+
const stream = r.ok ? (r.streaming ? "y" : "n") : "-"
|
| 141 |
+
console.log(`${tool} stream=${stream} ${r.provider.padEnd(8)} ${r.model}`)
|
| 142 |
+
}
|
| 143 |
+
const broken = results.filter((r) => !r.ok)
|
| 144 |
+
const noTool = results.filter((r) => r.ok && !r.toolCalled)
|
| 145 |
+
console.log(`\n${results.length} tested · ${broken.length} broken · ${noTool.length} no tool-call`)
|
| 146 |
+
if (broken.length) {
|
| 147 |
+
console.log("\nBROKEN:")
|
| 148 |
+
for (const b of broken) console.log(` ${b.provider}/${b.model}: ${b.error}`)
|
| 149 |
+
}
|