Update entrypoint.sh
Browse files- entrypoint.sh +3 -2
entrypoint.sh
CHANGED
|
@@ -11,7 +11,7 @@ REASONING_MODE="${REASONING_MODE:-off}"
|
|
| 11 |
CACHE_TYPE_K="${CACHE_TYPE_K:-q8_0}"
|
| 12 |
CACHE_TYPE_V="${CACHE_TYPE_V:-q8_0}"
|
| 13 |
FLASH_ATTN="${FLASH_ATTN:-on}"
|
| 14 |
-
SPEC_TYPE="${SPEC_TYPE:-
|
| 15 |
|
| 16 |
# NOTE: the b8840 CPU build does not support draft-mtp (needs a newer
|
| 17 |
# llama.cpp build than this image ships). ngram-cache is a self-speculation
|
|
@@ -32,4 +32,5 @@ exec /app/llama-server \
|
|
| 32 |
--cache-type-v "$CACHE_TYPE_V" \
|
| 33 |
--reasoning "$REASONING_MODE" \
|
| 34 |
-fa "$FLASH_ATTN" \
|
| 35 |
-
$SPEC_FLAGS
|
|
|
|
|
|
| 11 |
CACHE_TYPE_K="${CACHE_TYPE_K:-q8_0}"
|
| 12 |
CACHE_TYPE_V="${CACHE_TYPE_V:-q8_0}"
|
| 13 |
FLASH_ATTN="${FLASH_ATTN:-on}"
|
| 14 |
+
SPEC_TYPE="${SPEC_TYPE:-draft-mtp}"
|
| 15 |
|
| 16 |
# NOTE: the b8840 CPU build does not support draft-mtp (needs a newer
|
| 17 |
# llama.cpp build than this image ships). ngram-cache is a self-speculation
|
|
|
|
| 32 |
--cache-type-v "$CACHE_TYPE_V" \
|
| 33 |
--reasoning "$REASONING_MODE" \
|
| 34 |
-fa "$FLASH_ATTN" \
|
| 35 |
+
$SPEC_FLAGS \
|
| 36 |
+
--spec-draft-n-max 4
|