Spaces:
Running
Running
| /** | |
| * Model/provider probe for the tool-calling flow. | |
| * | |
| * Sends the request shape runMcpFlow sends (stream, tools, tool_choice auto, | |
| * max_tokens) with a generated ML Intern-style history — hf_jobs calls and | |
| * training-log tool results — scaled to a target prompt size, then records how | |
| * the SSE stream ends. Because the history is generated from a seed, the | |
| * `recall` task can be graded byte-for-byte: it asks for the job id and the | |
| * exact `step=100` log line of specific jobs buried in the context. | |
| * | |
| * Verdicts: | |
| * OK finished normally; recall (when asked) mostly correct | |
| * DEGRADED coherent transport but wrong/fabricated recall, or degenerate | |
| * output (repetition), or reasoning-only until the cap | |
| * SWALLOWED provider billed far more completion tokens than it delivered, | |
| * or a tool task ended with "stop" and no tool call | |
| * TRUNCATED stream ended without [DONE] or without a finish_reason | |
| * ERROR non-200 response or transport failure | |
| * | |
| * Usage: | |
| * npm run probe-model -- --model zai-org/GLM-5.3-Flash | |
| * npm run probe-model -- --model zai-org/GLM-5.3-Flash --provider together | |
| * npm run probe-model -- --model moonshotai/Kimi-K3 --size 300000 --task recall | |
| * npm run probe-model -- --model zai-org/GLM-5.3 --suite quick | |
| * | |
| * Suites (default `standard`): each provider the router lists for the model | |
| * (or --provider) runs every step. Sizes are prompt tokens; `%` of the | |
| * provider's whole context window, clamped so prompt plus reply still fits. | |
| * quick tool@100k, recall@300k | |
| * standard tool@100k, recall@300k, recall@75%, recall@90% | |
| * | |
| * Results go to --out (default .probe-results/): a results.jsonl, one .json | |
| * summary and one .sse raw capture per run. Needs OPENAI_API_KEY (an HF token) | |
| * in the env or .env.local, as the other scripts do. | |
| */ | |
| import { existsSync, mkdirSync, readFileSync, writeFileSync, appendFileSync } from "fs"; | |
| import { join } from "path"; | |
| import { Agent, fetch as undiciFetch } from "undici"; | |
| type Args = Record<string, string | true>; | |
| const args: Args = {}; | |
| { | |
| const argv = process.argv.slice(2); | |
| for (let i = 0; i < argv.length; i++) { | |
| const m = /^--([^=]+)(?:=(.*))?$/.exec(argv[i]); | |
| if (!m) continue; | |
| if (m[2] !== undefined) args[m[1]] = m[2]; | |
| else if (argv[i + 1] !== undefined && !argv[i + 1].startsWith("--")) args[m[1]] = argv[++i]; | |
| else args[m[1]] = true; | |
| } | |
| } | |
| const str = (k: string): string | undefined => | |
| typeof args[k] === "string" ? String(args[k]) : undefined; | |
| function loadEnv(): { baseUrl: string; apiKey: string } { | |
| const env = new Map<string, string>(); | |
| for (const file of [".env", ".env.local"]) { | |
| try { | |
| for (const line of readFileSync(file, "utf-8").split("\n")) { | |
| const match = line.match(/^([A-Z0-9_]+)=(.*)$/); | |
| if (match) env.set(match[1], match[2].replace(/^["']|["']$/g, "")); | |
| } | |
| } catch { | |
| // file optional | |
| } | |
| } | |
| const baseUrl = | |
| process.env.OPENAI_BASE_URL ?? env.get("OPENAI_BASE_URL") ?? "https://router.huggingface.co/v1"; | |
| const apiKey = process.env.OPENAI_API_KEY ?? env.get("OPENAI_API_KEY") ?? ""; | |
| if (!apiKey) throw new Error("No OPENAI_API_KEY found in env or .env.local"); | |
| return { baseUrl: baseUrl.replace(/\/$/, ""), apiKey }; | |
| } | |
| const { baseUrl, apiKey } = loadEnv(); | |
| const OUT = str("out") ?? ".probe-results"; | |
| mkdirSync(OUT, { recursive: true }); | |
| // Prefill of a 1M-token prompt can exceed undici's default 5-minute header | |
| // timeout; the probe measures that instead of dying on it. | |
| const agent = new Agent({ headersTimeout: 0, bodyTimeout: 0 }); | |
| type OaTool = { | |
| type: "function"; | |
| function: { name: string; description?: string; parameters?: Record<string, unknown> }; | |
| }; | |
| // The Hub MCP tool list depends on the account's settings; these three are what | |
| // the ML Intern prompt names, so they are always advertised (the mode's history | |
| // calls them, and a call to an unadvertised name is one of the failure modes | |
| // under test). | |
| const REQUIRED_TOOLS: OaTool[] = [ | |
| { | |
| type: "function", | |
| function: { | |
| name: "hf_jobs", | |
| description: | |
| "Remote compute for Hugging Face workflows. Run Python/UV or Docker jobs to analyze Hub datasets, repos, models and large files; run batch inference/evaluation; or perform long-running work. Includes submit, logs, inspect, cancel, schedule, and volume mounting.", | |
| parameters: { | |
| type: "object", | |
| properties: { | |
| operation: { | |
| type: "string", | |
| enum: ["run", "uv", "ps", "logs", "inspect", "cancel"], | |
| description: "Operation to execute.", | |
| }, | |
| args: { | |
| type: "object", | |
| description: "Operation-specific arguments as a JSON object", | |
| additionalProperties: {}, | |
| }, | |
| }, | |
| }, | |
| }, | |
| }, | |
| { | |
| type: "function", | |
| function: { | |
| name: "hf_sandbox_exec", | |
| description: | |
| "Run a shell command inside a Hugging Face Sandbox. Grammar: exec HANDLE SHELL_COMMAND [--workdir PATH] [--timeout SECONDS] [--detach]. SHELL_COMMAND is one string token and runs via /bin/sh -lc.", | |
| parameters: { | |
| type: "object", | |
| additionalProperties: false, | |
| properties: { | |
| cmd: { type: "string", enum: ["exec"] }, | |
| args: { type: "array", items: { type: "string" } }, | |
| }, | |
| required: ["cmd", "args"], | |
| }, | |
| }, | |
| }, | |
| { | |
| type: "function", | |
| function: { | |
| name: "hf_fs_write", | |
| description: | |
| "Write or remove files in Hugging Face repositories. Grammar: put URI [--base64] [-m MESSAGE] [--branch BRANCH] [--create-pr]; rm URI [-m MESSAGE]. URI must be an hf://models|datasets|spaces/OWNER/NAME/PATH file URI. Provide file data in the content field.", | |
| parameters: { | |
| type: "object", | |
| properties: { | |
| cmd: { type: "string", enum: ["put", "rm"] }, | |
| args: { type: "array", items: { type: "string" } }, | |
| content: { type: "string" }, | |
| }, | |
| required: ["cmd", "args"], | |
| }, | |
| }, | |
| }, | |
| ]; | |
| async function hubTools(): Promise<OaTool[]> { | |
| const cache = join(OUT, "hub-tools.json"); | |
| if (existsSync(cache)) return JSON.parse(readFileSync(cache, "utf-8")); | |
| const url = "https://hf.co/mcp?login"; | |
| const headers = { | |
| "content-type": "application/json", | |
| accept: "application/json, text/event-stream", | |
| authorization: `Bearer ${apiKey}`, | |
| }; | |
| const rpc = async (body: unknown, sid?: string) => { | |
| const res = await fetch(url, { | |
| method: "POST", | |
| headers: { ...headers, ...(sid ? { "mcp-session-id": sid } : {}) }, | |
| body: JSON.stringify(body), | |
| }); | |
| const text = await res.text(); | |
| let json: { | |
| result?: { | |
| tools?: { name: string; description?: string; inputSchema?: Record<string, unknown> }[]; | |
| }; | |
| } = {}; | |
| try { | |
| const data = (res.headers.get("content-type") ?? "").includes("event-stream") | |
| ? text | |
| .split("\n") | |
| .find((l) => l.startsWith("data:")) | |
| ?.slice(5) | |
| : text; | |
| if (data) json = JSON.parse(data); | |
| } catch { | |
| // a notification response has no body | |
| } | |
| return { res, json }; | |
| }; | |
| let listed: OaTool[] = []; | |
| try { | |
| const init = await rpc({ | |
| jsonrpc: "2.0", | |
| id: 1, | |
| method: "initialize", | |
| params: { | |
| protocolVersion: "2025-03-26", | |
| capabilities: {}, | |
| clientInfo: { name: "probe-model", version: "0" }, | |
| }, | |
| }); | |
| const sid = init.res.headers.get("mcp-session-id") ?? undefined; | |
| await rpc({ jsonrpc: "2.0", method: "notifications/initialized" }, sid); | |
| const list = await rpc({ jsonrpc: "2.0", id: 2, method: "tools/list" }, sid); | |
| listed = (list.json.result?.tools ?? []).map((t) => ({ | |
| type: "function" as const, | |
| function: { name: t.name, description: t.description, parameters: t.inputSchema }, | |
| })); | |
| } catch (e) { | |
| console.warn(`[probe] hub tools/list failed (${String(e)}); using the built-in schemas only`); | |
| } | |
| const names = new Set(listed.map((t) => t.function.name)); | |
| const tools = [...REQUIRED_TOOLS.filter((t) => !names.has(t.function.name)), ...listed]; | |
| writeFileSync(cache, JSON.stringify(tools, null, 1)); | |
| return tools; | |
| } | |
| // ---------- generated history ---------- | |
| function rng(seed: number) { | |
| let s = seed >>> 0; | |
| return () => { | |
| s = (s * 1664525 + 1013904223) >>> 0; | |
| return s / 4294967296; | |
| }; | |
| } | |
| const WORDS = | |
| "loss step epoch lr grad_norm eval accuracy tokens throughput checkpoint saved warmup batch shard dataset tokenizer optimizer adamw cosine schedule fp16 bf16 cuda memory allocated reserved wandb logged samples".split( | |
| " " | |
| ); | |
| function trainingScript(i: number): string { | |
| return `import torch, datasets | |
| from transformers import AutoModelForCausalLM, AutoTokenizer, Trainer, TrainingArguments | |
| ds = datasets.load_dataset("probe/ds-${i}", split="train") | |
| tok = AutoTokenizer.from_pretrained("Qwen/Qwen2.5-0.5B") | |
| model = AutoModelForCausalLM.from_pretrained("Qwen/Qwen2.5-0.5B", torch_dtype=torch.bfloat16) | |
| ds = ds.map(lambda ex: tok(ex["text"], truncation=True, max_length=1024), batched=True, remove_columns=ds.column_names) | |
| args = TrainingArguments(output_dir="out-${i}", per_device_train_batch_size=8, learning_rate=${(2e-5 + i * 1e-6).toExponential(2)}, num_train_epochs=1, bf16=True, logging_steps=10, save_steps=500, push_to_hub=True, hub_model_id="probe/run-${i}") | |
| Trainer(model=model, args=args, train_dataset=ds).train() | |
| `; | |
| } | |
| type Msg = Record<string, unknown> & { role: string }; | |
| const SYSTEM_PROMPT = [ | |
| "You are ML Intern, an assistant that runs machine-learning work on the Hugging Face Hub on the user's behalf.", | |
| "You launch training and evaluation as jobs with hf_jobs, inspect their logs, run quick commands in a sandbox with hf_sandbox_exec, and write files to repositories with hf_fs_write.", | |
| "Every job you launch is recorded in this conversation with its id and logs; refer back to them precisely and never invent ids or log lines.", | |
| "Ground every claim in the tool results present in the conversation.", | |
| ].join("\n"); | |
| const CHARS_PER_TOKEN = Number(str("cpt") ?? 2.05); | |
| function buildHistory(targetTokens: number, tools: OaTool[], seed: number) { | |
| const rand = rng(seed); | |
| const msgs: Msg[] = [{ role: "system", content: SYSTEM_PROMPT }]; | |
| let chars = SYSTEM_PROMPT.length + JSON.stringify(tools).length; | |
| const budget = targetTokens * CHARS_PER_TOKEN; | |
| let i = 0; | |
| while (chars < budget) { | |
| i += 1; | |
| const callId = `call_${i}_${Math.floor(rand() * 1e9).toString(36)}`; | |
| const jobId = Math.floor(rand() * 1e12).toString(16); | |
| const nlines = 40 + Math.floor(rand() * 80); | |
| const lines: string[] = []; | |
| for (let k = 0; k < nlines; k++) { | |
| const step = k * 10; | |
| const loss = (2.9 * Math.exp(-step / (400 + i * 37)) + rand() * 0.08).toFixed(4); | |
| const w = WORDS[Math.floor(rand() * WORDS.length)]; | |
| lines.push( | |
| `[${new Date(1756800000000 + i * 3600000 + k * 1500).toISOString()}] step=${step} loss=${loss} lr=${(2e-5 * (1 - step / 4000)).toExponential(3)} grad_norm=${(rand() * 3).toFixed(3)} tok/s=${Math.floor(9000 + rand() * 4000)} ${w}=${(rand() * 100).toFixed(2)}` | |
| ); | |
| } | |
| const round: Msg[] = [ | |
| { | |
| role: "user", | |
| content: | |
| i === 1 | |
| ? "Fine-tune Qwen2.5-0.5B on probe/ds-1 and report the loss curve." | |
| : `Now do the same for probe/ds-${i}, but with lr ${(2e-5 + i * 1e-6).toExponential(2)}.`, | |
| }, | |
| { | |
| role: "assistant", | |
| content: `Launching job ${i} on an a10g-small.`, | |
| tool_calls: [ | |
| { | |
| id: callId, | |
| type: "function", | |
| function: { | |
| name: "hf_jobs", | |
| arguments: JSON.stringify({ | |
| operation: "uv", | |
| args: { | |
| flavor: "a10g-small", | |
| timeout: "2h", | |
| script: trainingScript(i), | |
| secrets: ["HF_TOKEN"], | |
| }, | |
| }), | |
| }, | |
| }, | |
| ], | |
| }, | |
| { | |
| role: "tool", | |
| tool_call_id: callId, | |
| content: `Job ${jobId} launched. Status: COMPLETED\n--- logs ---\n${lines.join("\n")}\n--- end ---`, | |
| }, | |
| { | |
| role: "assistant", | |
| content: `Job ${i} (id \`${jobId}\`) finished. Final loss ${(0.3 + rand() * 0.2).toFixed(4)} after ${nlines * 10} steps; throughput ~${Math.floor(9000 + rand() * 4000)} tok/s. Model pushed to probe/run-${i}.`, | |
| }, | |
| ]; | |
| for (const m of round) { | |
| msgs.push(m); | |
| chars += JSON.stringify(m).length; | |
| } | |
| } | |
| return { msgs, rounds: i }; | |
| } | |
| const RECALL_JOBS = [5, 100, 148, 149, 180, 214]; | |
| const TASKS: Record<string, string> = { | |
| answer: | |
| "Write a detailed retrospective (at least 1500 words) of everything we did in this session: one section per job with the job id, learning rate, final loss, and what changed from the previous run, then a markdown table of all jobs, then recommendations for the next 5 experiments.", | |
| tool: "Now launch one more job on probe/ds-next with lr 3e-5 using hf_jobs. Write the full training script inline in the tool call like before.", | |
| bigtool: | |
| "Now launch one more job on probe/ds-next with hf_jobs. The script must be a complete, self-contained training script of at least 300 lines: a custom data collator, a cosine scheduler with warmup, periodic evaluation, checkpoint resumption, W&B logging, and extensive docstrings. Put the whole script inline in the tool call arguments.", | |
| dump: "Produce a complete audit document. First a markdown table of EVERY job in this session (id, dataset, learning rate, final loss, steps, throughput). Then, for EVERY job in order, a section with its id and the first 12 log lines reproduced verbatim in a code block. Do not skip or summarize any job; this document must be exhaustive even if it is very long.", | |
| recall: `Answer ONLY with a JSON object, no prose. For each of jobs ${RECALL_JOBS.join(", ")} (the Nth job we launched in this session): give the job id (the hex id from the tool result) and reproduce VERBATIM the log line containing step=100 from that job's logs as they appear in the tool result. Format: {"5": {"id": "...", "step100": "..."}, ...}. If a job's logs are genuinely not present in the context, use "NOT_PRESENT" for that field.`, | |
| }; | |
| // ---------- one run ---------- | |
| interface RunSpec { | |
| model: string; | |
| provider?: string; | |
| size: number; | |
| task: string; | |
| maxTokens: number; | |
| label: string; | |
| } | |
| interface RunResult extends RunSpec { | |
| modelId: string; | |
| status?: number; | |
| providerHeader?: string | null; | |
| ttfbMs?: number; | |
| totalMs?: number; | |
| chunks: number; | |
| maxGapMs: number; | |
| contentChars: number; | |
| reasoningChars: number; | |
| toolCalls: { name: string; argChars: number; valid: boolean }[]; | |
| finishReasons: string[]; | |
| sawDone: boolean; | |
| errors: string[]; | |
| streamError?: string; | |
| errorBody?: string; | |
| usage?: { | |
| prompt_tokens?: number; | |
| completion_tokens?: number; | |
| prompt_tokens_details?: { cached_tokens?: number }; | |
| }; | |
| recall?: { score: number; total: number; detail: string[] }; | |
| verdict: string; | |
| reason: string; | |
| contentTail: string; | |
| } | |
| function gradeRecall(content: string, history: Msg[]) { | |
| const tools = history.filter((m) => m.role === "tool"); | |
| const present = RECALL_JOBS.filter((n) => tools[n - 1]); | |
| let answer: Record<string, { id?: string; step100?: string }> = {}; | |
| try { | |
| answer = JSON.parse(/\{[\s\S]*\}/.exec(content)?.[0] ?? "{}"); | |
| } catch { | |
| // unparseable answers score zero | |
| } | |
| let score = 0; | |
| const detail = present.map((n) => { | |
| const text = String(tools[n - 1].content); | |
| const id = /Job ([0-9a-f]+) launched/.exec(text)?.[1]; | |
| const line = text.split("\n").find((l) => / step=100 /.test(l)); | |
| const a = answer[String(n)] ?? {}; | |
| const good = a.id === id && a.step100 === line; | |
| if (good) score++; | |
| return `${n}:${good ? "ok" : a.step100 === "NOT_PRESENT" ? "not-present" : "wrong"}`; | |
| }); | |
| return { score, total: present.length, detail }; | |
| } | |
| function uniqueWordRatio(text: string): number { | |
| const words = text.split(/\s+/).filter(Boolean); | |
| if (words.length < 50) return 1; | |
| return new Set(words).size / words.length; | |
| } | |
| async function runOne(spec: RunSpec, tools: OaTool[]): Promise<RunResult> { | |
| const { msgs, rounds } = buildHistory(spec.size, tools, 42); | |
| msgs.push({ role: "user", content: TASKS[spec.task] }); | |
| const modelId = spec.provider ? `${spec.model}:${spec.provider}` : spec.model; | |
| const body = { | |
| model: modelId, | |
| stream: true, | |
| messages: msgs, | |
| tools, | |
| tool_choice: "auto", | |
| max_tokens: spec.maxTokens, | |
| stream_options: { include_usage: true }, | |
| }; | |
| const result: RunResult = { | |
| ...spec, | |
| modelId, | |
| chunks: 0, | |
| maxGapMs: 0, | |
| contentChars: 0, | |
| reasoningChars: 0, | |
| toolCalls: [], | |
| finishReasons: [], | |
| sawDone: false, | |
| errors: [], | |
| verdict: "ERROR", | |
| reason: "", | |
| contentTail: "", | |
| }; | |
| void rounds; | |
| const raw: string[] = []; | |
| const t0 = Date.now(); | |
| let res: Awaited<ReturnType<typeof undiciFetch>>; | |
| try { | |
| res = await undiciFetch(`${baseUrl}/chat/completions`, { | |
| method: "POST", | |
| headers: { | |
| "content-type": "application/json", | |
| authorization: `Bearer ${apiKey}`, | |
| "X-use-cache": "false", | |
| }, | |
| body: JSON.stringify(body), | |
| dispatcher: agent, | |
| }); | |
| } catch (e) { | |
| result.streamError = String(e); | |
| result.reason = "fetch failed"; | |
| return result; | |
| } | |
| result.status = res.status; | |
| result.providerHeader = res.headers.get("x-inference-provider"); | |
| result.ttfbMs = Date.now() - t0; | |
| if (!res.ok || !res.body) { | |
| result.errorBody = (await res.text()).slice(0, 2000); | |
| result.reason = `HTTP ${res.status}`; | |
| return result; | |
| } | |
| let content = ""; | |
| let reasoning = ""; | |
| const calls: Record<number, { name: string; args: string }> = {}; | |
| let buf = ""; | |
| let lastAt = Date.now(); | |
| const dec = new TextDecoder(); | |
| try { | |
| for await (const piece of res.body as AsyncIterable<Uint8Array>) { | |
| const now = Date.now(); | |
| result.maxGapMs = Math.max(result.maxGapMs, now - lastAt); | |
| lastAt = now; | |
| const text = dec.decode(piece, { stream: true }); | |
| raw.push(text); | |
| buf += text; | |
| let idx: number; | |
| while ((idx = buf.indexOf("\n\n")) !== -1) { | |
| const evt = buf.slice(0, idx); | |
| buf = buf.slice(idx + 2); | |
| for (const line of evt.split("\n")) { | |
| if (!line.startsWith("data:")) continue; | |
| const data = line.slice(5).trim(); | |
| if (data === "[DONE]") { | |
| result.sawDone = true; | |
| continue; | |
| } | |
| let j: Record<string, unknown>; | |
| try { | |
| j = JSON.parse(data); | |
| } catch { | |
| continue; | |
| } | |
| result.chunks++; | |
| if (j.error) result.errors.push(JSON.stringify(j.error).slice(0, 500)); | |
| if (j.usage) result.usage = j.usage as RunResult["usage"]; | |
| const choice = ( | |
| j.choices as { finish_reason?: string; delta?: Record<string, unknown> }[] | undefined | |
| )?.[0]; | |
| if (!choice) continue; | |
| if (choice.finish_reason) result.finishReasons.push(choice.finish_reason); | |
| const d = choice.delta ?? {}; | |
| if (typeof d.content === "string") content += d.content; | |
| const r = d.reasoning_content ?? d.reasoning ?? d.reasoning_text; | |
| if (typeof r === "string") reasoning += r; | |
| for (const tc of (d.tool_calls as { | |
| index?: number; | |
| function?: { name?: string; arguments?: string }; | |
| }[]) ?? []) { | |
| const cur = (calls[tc.index ?? 0] ??= { name: "", args: "" }); | |
| if (tc.function?.name) cur.name = tc.function.name; | |
| if (tc.function?.arguments) cur.args += tc.function.arguments; | |
| } | |
| } | |
| } | |
| } | |
| } catch (e) { | |
| result.streamError = String(e); | |
| } | |
| result.totalMs = Date.now() - t0; | |
| result.contentChars = content.length; | |
| result.reasoningChars = reasoning.length; | |
| result.contentTail = content.slice(-300); | |
| result.toolCalls = Object.values(calls).map((c) => { | |
| let valid = true; | |
| try { | |
| JSON.parse(c.args); | |
| } catch { | |
| valid = false; | |
| } | |
| return { name: c.name, argChars: c.args.length, valid }; | |
| }); | |
| if (spec.task === "recall") result.recall = gradeRecall(content, msgs); | |
| writeFileSync(join(OUT, `${spec.label}.sse`), raw.join("")); | |
| // ---- verdict ---- | |
| const finish = result.finishReasons.at(-1); | |
| const deliveredTokens = | |
| (content.length + | |
| reasoning.length + | |
| Object.values(calls).reduce((n, c) => n + c.args.length, 0)) / | |
| 3; | |
| const billed = result.usage?.completion_tokens ?? 0; | |
| if (result.streamError) { | |
| result.verdict = "ERROR"; | |
| result.reason = result.streamError; | |
| } else if (result.errors.length) { | |
| result.verdict = "ERROR"; | |
| result.reason = result.errors[0]; | |
| } else if (!result.sawDone || !finish) { | |
| result.verdict = "TRUNCATED"; | |
| result.reason = !result.sawDone ? "stream ended without [DONE]" : "no finish_reason"; | |
| } else if ( | |
| ["tool", "bigtool"].includes(spec.task) && | |
| finish === "stop" && | |
| result.toolCalls.length === 0 | |
| ) { | |
| result.verdict = "SWALLOWED"; | |
| result.reason = `tool task ended with "stop" and no tool call (${billed} tokens billed)`; | |
| } else if (billed > 200 && billed > deliveredTokens * 2 + 100) { | |
| result.verdict = "SWALLOWED"; | |
| result.reason = `billed ${billed} completion tokens, delivered ~${Math.round(deliveredTokens)}`; | |
| } else if (result.toolCalls.some((c) => !c.valid) && finish !== "length") { | |
| result.verdict = "DEGRADED"; | |
| result.reason = "tool call arguments are not valid JSON"; | |
| } else if (uniqueWordRatio(content) < 0.3) { | |
| result.verdict = "DEGRADED"; | |
| result.reason = "degenerate repetition in output"; | |
| } else if (spec.task === "recall" && result.recall && content.trim().length === 0) { | |
| result.verdict = "DEGRADED"; | |
| result.reason = | |
| finish === "length" ? "reasoning-only until the output cap; no answer" : "no visible answer"; | |
| } else if ( | |
| spec.task === "recall" && | |
| result.recall && | |
| result.recall.score * 2 < result.recall.total | |
| ) { | |
| result.verdict = "DEGRADED"; | |
| result.reason = `recall ${result.recall.score}/${result.recall.total} [${result.recall.detail.join(" ")}]`; | |
| } else { | |
| result.verdict = "OK"; | |
| result.reason = | |
| spec.task === "recall" && result.recall | |
| ? `recall ${result.recall.score}/${result.recall.total}` | |
| : result.toolCalls.length | |
| ? `tool call ${result.toolCalls.map((c) => `${c.name}(${c.argChars} chars)`).join(", ")}` | |
| : `finish=${finish}, ${content.length} chars`; | |
| } | |
| return result; | |
| } | |
| // ---------- suites ---------- | |
| async function providersFor( | |
| model: string | |
| ): Promise<{ provider: string; context_length?: number }[]> { | |
| const res = await fetch(`${baseUrl}/models`); | |
| const json = (await res.json()) as { | |
| data: { | |
| id: string; | |
| providers?: { provider: string; context_length?: number; status?: string }[]; | |
| }[]; | |
| }; | |
| const entry = json.data.find((m) => m.id === model); | |
| if (!entry) throw new Error(`model ${model} not in ${baseUrl}/models`); | |
| return (entry.providers ?? []).filter((p) => p.status !== "error"); | |
| } | |
| const SUITES: Record<string, { task: string; size: string }[]> = { | |
| quick: [ | |
| { task: "tool", size: "100000" }, | |
| { task: "recall", size: "300000" }, | |
| ], | |
| standard: [ | |
| { task: "tool", size: "100000" }, | |
| { task: "recall", size: "300000" }, | |
| { task: "recall", size: "75%" }, | |
| { task: "recall", size: "90%" }, | |
| ], | |
| }; | |
| function resolveSize( | |
| size: string, | |
| contextLength: number | undefined, | |
| maxTokens: number | |
| ): number | undefined { | |
| if (size.endsWith("%")) { | |
| const window = contextLength ?? 131072; | |
| const pct = Number(size.slice(0, -1)) / 100; | |
| // Of the whole window, clamped only so prompt plus reply still fits. | |
| return Math.min(Math.floor(window * pct), window - maxTokens); | |
| } | |
| const n = Number(size); | |
| if (contextLength && n > contextLength - maxTokens) return undefined; | |
| return n; | |
| } | |
| function line(r: RunResult): string { | |
| const prompt = r.usage?.prompt_tokens?.toLocaleString("en-US") ?? "-"; | |
| return [ | |
| r.verdict.padEnd(9), | |
| r.modelId.padEnd(44), | |
| `${r.task}@${prompt}`.padEnd(20), | |
| `finish=${r.finishReasons.at(-1) ?? "-"}`.padEnd(18), | |
| `ttfb=${r.ttfbMs ?? "-"}ms gap=${r.maxGapMs}ms`.padEnd(28), | |
| r.reason, | |
| ].join(" "); | |
| } | |
| async function main() { | |
| const model = str("model"); | |
| if (!model) { | |
| console.error( | |
| "usage: npm run probe-model -- --model <id> [--provider p] [--suite quick|standard] [--size N|N%] [--task recall|tool|answer|bigtool|dump] [--max-tokens N] [--out dir]" | |
| ); | |
| process.exit(1); | |
| } | |
| const maxTokens = Number(str("max-tokens") ?? 32768); | |
| const tools = await hubTools(); | |
| const listed = await providersFor(model); | |
| const providers = str("provider") | |
| ? [ | |
| { | |
| provider: String(str("provider")), | |
| context_length: listed.find((p) => p.provider === str("provider"))?.context_length, | |
| }, | |
| ] | |
| : listed; | |
| const steps = | |
| str("size") || str("task") | |
| ? [{ task: str("task") ?? "recall", size: str("size") ?? "300000" }] | |
| : SUITES[str("suite") ?? "standard"]; | |
| if (!steps) throw new Error(`unknown suite ${str("suite")}`); | |
| console.log( | |
| `model ${model}; providers ${providers.map((p) => p.provider).join(", ")}; ${tools.length} tools; out ${OUT}` | |
| ); | |
| const results: RunResult[] = []; | |
| for (const p of providers) { | |
| for (const step of steps) { | |
| const size = resolveSize(step.size, p.context_length, maxTokens); | |
| if (size === undefined) { | |
| console.log( | |
| `SKIP ${model}:${p.provider} ${step.task}@${step.size} exceeds the provider window` | |
| ); | |
| continue; | |
| } | |
| const label = `${model.replace("/", "_")}__${p.provider}__${step.task}@${size}`; | |
| const spec: RunSpec = { | |
| model, | |
| provider: p.provider, | |
| size, | |
| task: step.task, | |
| maxTokens, | |
| label, | |
| }; | |
| const r = await runOne(spec, tools); | |
| results.push(r); | |
| appendFileSync(join(OUT, "results.jsonl"), JSON.stringify(r) + "\n"); | |
| writeFileSync(join(OUT, `${label}.json`), JSON.stringify(r, null, 2)); | |
| console.log(line(r)); | |
| } | |
| } | |
| const bad = results.filter((r) => r.verdict !== "OK"); | |
| console.log(`\n${results.length} runs, ${bad.length} not OK`); | |
| process.exit(bad.length ? 2 : 0); | |
| } | |
| main().catch((e) => { | |
| console.error(e); | |
| process.exit(1); | |
| }); | |