| set -euo pipefail | |
| ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" | |
| MODEL="${MODEL:-$ROOT/models/Ling-3.0-tiny-Q4_K_M.gguf}" | |
| BIN="${BIN:-$ROOT/llama.cpp/build/bin/sdq-chat}" | |
| RAM_BUDGET_MB="${RAM_BUDGET_MB:-4096}" | |
| RESERVE_MB="${RESERVE_MB:-700}" | |
| CTX="${CTX:-256}" | |
| THREADS="${THREADS:-4}" | |
| UBATCH="${UBATCH:-32}" | |
| [[ -x "$BIN" ]] || { printf 'Missing %s; build the engine as documented in README.md\n' "$BIN" >&2; exit 1; } | |
| [[ -s "$MODEL" ]] || { printf 'Missing model: %s\n' "$MODEL" >&2; exit 1; } | |
| export SDQ_MODEL_PATH="$MODEL" | |
| export SDQ_RAM_BUDGET_MB="$RAM_BUDGET_MB" | |
| export SDQ_RESERVE_MB="$RESERVE_MB" | |
| export SDQ_PREFETCH=0 | |
| export SDQ_FADV_DONTNEED=1 | |
| export SDQ_IO_THREADS="$((THREADS * 2))" | |
| export LLAMA_ARG_N_GPU_LAYERS=0 | |
| exec "$BIN" -m "$MODEL" -c "$CTX" -t "$THREADS" -b "$UBATCH" \ | |
| -p "${PROMPT:-What is the capital of France?}" -n "${N_PREDICT:-48}" \ | |
| --temp 0 --no-interactive --ram-budget-mb "$RAM_BUDGET_MB" | |