File size: 954 Bytes
2b6ab7c | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 | #!/usr/bin/env bash
set -euo pipefail
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
MODEL="${MODEL:-$ROOT/models/Ling-3.0-tiny-Q4_K_M.gguf}"
BIN="${BIN:-$ROOT/llama.cpp/build/bin/sdq-chat}"
RAM_BUDGET_MB="${RAM_BUDGET_MB:-4096}"
RESERVE_MB="${RESERVE_MB:-700}"
CTX="${CTX:-256}"
THREADS="${THREADS:-4}"
UBATCH="${UBATCH:-32}"
[[ -x "$BIN" ]] || { printf 'Missing %s; build the engine as documented in README.md\n' "$BIN" >&2; exit 1; }
[[ -s "$MODEL" ]] || { printf 'Missing model: %s\n' "$MODEL" >&2; exit 1; }
export SDQ_MODEL_PATH="$MODEL"
export SDQ_RAM_BUDGET_MB="$RAM_BUDGET_MB"
export SDQ_RESERVE_MB="$RESERVE_MB"
export SDQ_PREFETCH=0
export SDQ_FADV_DONTNEED=1
export SDQ_IO_THREADS="$((THREADS * 2))"
export LLAMA_ARG_N_GPU_LAYERS=0
exec "$BIN" -m "$MODEL" -c "$CTX" -t "$THREADS" -b "$UBATCH" \
-p "${PROMPT:-What is the capital of France?}" -n "${N_PREDICT:-48}" \
--temp 0 --no-interactive --ram-budget-mb "$RAM_BUDGET_MB"
|