ramedde commited on
Commit
9955d2b
·
verified ·
1 Parent(s): 3e7bfd4

Update entrypoint.sh

Browse files
Files changed (1) hide show
  1. entrypoint.sh +1 -34
entrypoint.sh CHANGED
@@ -31,38 +31,5 @@ exec /app/llama-server \
31
  --cache-type-k "$CACHE_TYPE_K" \
32
  --cache-type-v "$CACHE_TYPE_V" \
33
  --reasoning "$REASONING_MODE" \
34
- -fa "$FLASH_ATTN" \
35
- $SPEC_FLAGS#!/bin/sh
36
- set -eu
37
-
38
- MODEL_SPEC="${MODEL_SPEC:-unsloth/gemma-4-E2B-it-qat-GGUF:UD-Q4_K_XL}"
39
- HOST="${HOST:-0.0.0.0}"
40
- PORT="${PORT:-7860}"
41
- CTX_SIZE="${CTX_SIZE:-8192}"
42
- THREADS="${THREADS:-2}"
43
- PARALLEL="${PARALLEL:-1}"
44
- REASONING_MODE="${REASONING_MODE:-off}"
45
- CACHE_TYPE_K="${CACHE_TYPE_K:-q8_0}"
46
- CACHE_TYPE_V="${CACHE_TYPE_V:-q8_0}"
47
- FLASH_ATTN="${FLASH_ATTN:-on}"
48
- SPEC_TYPE="${SPEC_TYPE:-ngram-cache}"
49
-
50
- # NOTE: the b8840 CPU build does not support draft-mtp (needs a newer
51
- # llama.cpp build than this image ships). ngram-cache is a self-speculation
52
- # mode that needs no external draft model: it speculates from n-grams already
53
- # seen in context/output, verified by the target model, so it's zero-cost on
54
- # accuracy. Set SPEC_TYPE=none in Space secrets to disable entirely.
55
- SPEC_FLAGS="--spec-type $SPEC_TYPE"
56
 
57
- exec /app/llama-server \
58
- -hf "$MODEL_SPEC" \
59
- --host "$HOST" \
60
- --port "$PORT" \
61
- --ctx-size "$CTX_SIZE" \
62
- --threads "$THREADS" \
63
- --threads-batch "$THREADS" \
64
- --parallel "$PARALLEL" \
65
- --cache-type-k "$CACHE_TYPE_K" \
66
- --cache-type-v "$CACHE_TYPE_V" \
67
- --reasoning "$REASONING_MODE" \
68
- -fa "$FLASH_ATTN"
 
31
  --cache-type-k "$CACHE_TYPE_K" \
32
  --cache-type-v "$CACHE_TYPE_V" \
33
  --reasoning "$REASONING_MODE" \
34
+ -fa "$FLASH_ATTN"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
35