File size: 6,929 Bytes
2e81416 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 | #!/usr/bin/env bash
# Janus-35B — smoke test against a running Ollama daemon.
#
# Verifies:
# 1. The Ollama server is reachable.
# 2. The target model is loaded / loadable.
# 3. The model exposes the `tools` capability (Modelfile TEMPLATE wired).
# 4. A single chat round-trip succeeds and produces non-empty output.
# 5. No chat-template control tokens leak into the response.
# 6. (TOOLS_TEST=1) An end-to-end tool-call round-trip emits a structured
# tool_calls array with the expected name and arguments. Off by default
# because it costs ~5-10 sec of inference; on for comprehensive runs.
#
# Usage:
# ./scripts/smoke_test.sh # fast checks only
# TOOLS_TEST=1 ./scripts/smoke_test.sh # add tool-call round-trip
# MODEL=hf.co/FoolDev/Janus-35B-HERETIC:Q4_K_M ./scripts/smoke_test.sh
# HOST=http://localhost:11434 ./scripts/smoke_test.sh
#
# Requires: curl, jq.
set -euo pipefail
MODEL="${MODEL:-janus}"
HOST="${HOST:-http://localhost:11434}"
PROMPT="${PROMPT:-Reply with the single word: OK}"
require() {
if ! command -v "$1" >/dev/null 2>&1; then
echo "[!] missing dependency: $1" >&2; exit 1
fi
}
require curl
require jq
echo "[*] host: ${HOST}"
echo "[*] model: ${MODEL}"
# ---- 1. Server up? ----------------------------------------------------------
if ! curl -fsS "${HOST}/api/tags" >/dev/null; then
echo "[!] Ollama not reachable at ${HOST}. Is 'ollama serve' running?" >&2
exit 1
fi
echo "[+] server reachable"
# ---- 2. Model present? ------------------------------------------------------
# Match case-insensitively: Ollama 0.24 normalizes model names at lookup but
# preserves whatever case was first registered on disk (e.g. an earlier
# session may leave a `Janus-35B:latest` manifest dir behind even when a
# build was invoked with TAG=janus). The exact tag the user typed is still
# valid for `ollama run` — the comparison just needs to be case-folded to
# match.
if ! curl -fsS "${HOST}/api/tags" | jq -e --arg m "${MODEL}" '.models[] | select((.name | ascii_downcase) | startswith($m | ascii_downcase))' >/dev/null; then
echo "[!] Model '${MODEL}' not found. Build it first:" >&2
echo " ./scripts/build.sh # Q4_K_M" >&2
echo " ./scripts/build.sh Q3_K_M # smaller quant" >&2
echo " ./scripts/load_bundle.sh # load this repo's qwen35moe bundle" >&2
exit 1
fi
echo "[+] model present"
# ---- 3. Capability guard ----------------------------------------------------
# The Modelfile TEMPLATE must expose .Tools / .ToolCalls so Ollama lists
# `tools` under capabilities. Without it, /api/chat with a tools array returns
# 400 "does not support tools" even though plain chat works. Catches Modelfile
# regressions that strip or break the TEMPLATE.
CAPS="$(curl -fsS "${HOST}/api/show" -H 'Content-Type: application/json' \
-d "$(jq -n --arg m "${MODEL}" '{name: $m}')" | jq -r '.capabilities[]?')"
if ! grep -qx -- 'tools' <<<"${CAPS}"; then
echo "[!] model missing capability: tools" >&2
echo " Modelfile likely missing TEMPLATE that references .Tools / .ToolCalls." >&2
echo "----- present capabilities -----" >&2
echo "${CAPS:-<none>}" >&2
echo "--------------------------------" >&2
exit 1
fi
echo "[+] capabilities include: tools"
# ---- 4. Round-trip ----------------------------------------------------------
echo "[*] sending test prompt..."
RESP="$(curl -fsS "${HOST}/api/chat" \
-H 'Content-Type: application/json' \
-d "$(jq -n --arg m "${MODEL}" --arg p "${PROMPT}" '{
model: $m,
messages: [{role:"user", content:$p}],
stream: false
}')" | jq -r '.message.content // empty')"
if [[ -z "${RESP}" ]]; then
echo "[!] empty response from model" >&2
exit 1
fi
# Token-leakage guard: if any of the chat-template control tokens show up
# verbatim in the response, the Modelfile stop-token list is broken and the
# model is bleeding past EOS.
LEAKED=()
for tok in '<|im_start|>' '<|im_end|>' '<|endoftext|>'; do
if grep -qF -- "${tok}" <<<"${RESP}"; then
LEAKED+=("${tok}")
fi
done
if (( ${#LEAKED[@]} )); then
echo "[!] response contains raw control tokens: ${LEAKED[*]}" >&2
echo " Modelfile likely missing PARAMETER stop directives." >&2
echo "----- model said -----" >&2
echo "${RESP}" >&2
echo "----------------------" >&2
exit 1
fi
echo "[+] round-trip OK"
echo "----- model said -----"
echo "${RESP}"
echo "----------------------"
# ---- 5. Tool-call round-trip (opt-in via TOOLS_TEST=1) ----------------------
#
# Capability advertisement (step 3) only checks the TEMPLATE references
# .Tools / .ToolCalls. It does NOT check the model actually emits a parseable
# tool call. A regression in the prompt scaffolding (e.g. the system-prompt
# instructions inside the TEMPLATE going stale) can leave capabilities
# reported correctly but tool calls failing — the assistant prose-describes
# the call instead of emitting <tool_call>{...}</tool_call>. This block sends
# a tools-array request, parses .message.tool_calls, and asserts the shape
# matches.
if [[ "${TOOLS_TEST:-0}" == "1" ]]; then
echo "[*] tool-call round-trip..."
TOOL_RESP="$(curl -fsS "${HOST}/api/chat" \
-H 'Content-Type: application/json' \
-d "$(jq -n --arg m "${MODEL}" '{
model: $m,
messages: [{role:"user", content:"Call get_weather for Tokyo. Respond ONLY with the tool call."}],
tools: [{
type: "function",
function: {
name: "get_weather",
description: "Get the weather for a city",
parameters: {
type: "object",
properties: {city: {type: "string"}},
required: ["city"]
}
}
}],
stream: false,
options: {num_predict: 1024, temperature: 0.3}
}')")"
TC_COUNT="$(jq -r '.message.tool_calls // [] | length' <<<"${TOOL_RESP}")"
if [[ "${TC_COUNT}" -lt 1 ]]; then
echo "[!] model did not emit a tool call" >&2
echo "----- response -----" >&2
echo "${TOOL_RESP}" | jq . >&2
echo "--------------------" >&2
exit 1
fi
TC_NAME="$(jq -r '.message.tool_calls[0].function.name // empty' <<<"${TOOL_RESP}")"
TC_CITY="$(jq -r '.message.tool_calls[0].function.arguments.city // empty' <<<"${TOOL_RESP}")"
if [[ "${TC_NAME}" != "get_weather" ]]; then
echo "[!] unexpected tool name: '${TC_NAME}' (wanted 'get_weather')" >&2
exit 1
fi
if [[ "${TC_CITY,,}" != "tokyo" ]]; then
echo "[!] unexpected city argument: '${TC_CITY}' (wanted 'Tokyo' case-insensitive)" >&2
exit 1
fi
echo "[+] tool-call round-trip OK (name=${TC_NAME} city=${TC_CITY})"
fi
|