Update entrypoint.sh
Browse files- entrypoint.sh +1 -34
entrypoint.sh
CHANGED
|
@@ -31,38 +31,5 @@ exec /app/llama-server \
|
|
| 31 |
--cache-type-k "$CACHE_TYPE_K" \
|
| 32 |
--cache-type-v "$CACHE_TYPE_V" \
|
| 33 |
--reasoning "$REASONING_MODE" \
|
| 34 |
-
-fa "$FLASH_ATTN"
|
| 35 |
-
$SPEC_FLAGS#!/bin/sh
|
| 36 |
-
set -eu
|
| 37 |
-
|
| 38 |
-
MODEL_SPEC="${MODEL_SPEC:-unsloth/gemma-4-E2B-it-qat-GGUF:UD-Q4_K_XL}"
|
| 39 |
-
HOST="${HOST:-0.0.0.0}"
|
| 40 |
-
PORT="${PORT:-7860}"
|
| 41 |
-
CTX_SIZE="${CTX_SIZE:-8192}"
|
| 42 |
-
THREADS="${THREADS:-2}"
|
| 43 |
-
PARALLEL="${PARALLEL:-1}"
|
| 44 |
-
REASONING_MODE="${REASONING_MODE:-off}"
|
| 45 |
-
CACHE_TYPE_K="${CACHE_TYPE_K:-q8_0}"
|
| 46 |
-
CACHE_TYPE_V="${CACHE_TYPE_V:-q8_0}"
|
| 47 |
-
FLASH_ATTN="${FLASH_ATTN:-on}"
|
| 48 |
-
SPEC_TYPE="${SPEC_TYPE:-ngram-cache}"
|
| 49 |
-
|
| 50 |
-
# NOTE: the b8840 CPU build does not support draft-mtp (needs a newer
|
| 51 |
-
# llama.cpp build than this image ships). ngram-cache is a self-speculation
|
| 52 |
-
# mode that needs no external draft model: it speculates from n-grams already
|
| 53 |
-
# seen in context/output, verified by the target model, so it's zero-cost on
|
| 54 |
-
# accuracy. Set SPEC_TYPE=none in Space secrets to disable entirely.
|
| 55 |
-
SPEC_FLAGS="--spec-type $SPEC_TYPE"
|
| 56 |
|
| 57 |
-
exec /app/llama-server \
|
| 58 |
-
-hf "$MODEL_SPEC" \
|
| 59 |
-
--host "$HOST" \
|
| 60 |
-
--port "$PORT" \
|
| 61 |
-
--ctx-size "$CTX_SIZE" \
|
| 62 |
-
--threads "$THREADS" \
|
| 63 |
-
--threads-batch "$THREADS" \
|
| 64 |
-
--parallel "$PARALLEL" \
|
| 65 |
-
--cache-type-k "$CACHE_TYPE_K" \
|
| 66 |
-
--cache-type-v "$CACHE_TYPE_V" \
|
| 67 |
-
--reasoning "$REASONING_MODE" \
|
| 68 |
-
-fa "$FLASH_ATTN"
|
|
|
|
| 31 |
--cache-type-k "$CACHE_TYPE_K" \
|
| 32 |
--cache-type-v "$CACHE_TYPE_V" \
|
| 33 |
--reasoning "$REASONING_MODE" \
|
| 34 |
+
-fa "$FLASH_ATTN"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 35 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|