fix flash attention

README.md

@@ -37,6 +37,8 @@ Optional tuning variables:
 - `ENABLE_TOOLS`: set to `1` to enable llama.cpp shell tool support when `LANGSEARCH_API_KEY` is present, default `0`
 - `HTTP_THREADS`: HTTP server worker threads, default `1` for personal single-request inference
 - `LOG_VERBOSITY`: llama.cpp log verbosity, default `1` to reduce runtime logging overhead
+- `MMAP`: set to `1` to use memory-mapped model loading, default `0`
+- `FLASH_ATTN`: set to `0` to disable flash attention if a model hangs or fails during startup, default `1`
 - `NO_WARMUP`: set to `1` to skip model warmup and save startup time
 - `PORT`: listen port, default `7860`
 
@@ -71,6 +73,7 @@ For small 2B models on CPU-only inference (~2-4 t/s baseline), optimize these se
 - `CACHE_TYPE_K`, `CACHE_TYPE_V`: Try `q4_0` instead of `f16` to reduce KV cache bandwidth and memory use at longer contexts
 - `ENABLE_TOOLS`: Leave as `0` unless you need tool calling. Built-in shell tools add prompt/tool overhead and should not be exposed publicly.
 - `MODEL_FILE`: If the repo has multiple GGUFs, test a lighter quant such as `Q4_0`, `Q4_K_S`, or `Q3_K_M` for speed/quality tradeoffs.
+- Compatibility: if a GGUF hangs during startup, try `FLASH_ATTN=0`, `CACHE_TYPE_K=f16`, `CACHE_TYPE_V=f16`, and then `MMAP=1`.
 
 **Example for 2B model (CPU-only):**
 ```

start.sh

@@ -93,6 +93,8 @@ export TOOLS="${TOOLS:-}"
 export ENABLE_TOOLS="${ENABLE_TOOLS:-0}"
 export HTTP_THREADS="${HTTP_THREADS:-1}"
 export LOG_VERBOSITY="${LOG_VERBOSITY:-1}"
+export MMAP="${MMAP:-0}"
+export FLASH_ATTN="${FLASH_ATTN:-1}"
 export NO_WARMUP="${NO_WARMUP:-0}"
 export LANGSEARCH_API_KEY="${LANGSEARCH_API_KEY:-}"
 
@@ -169,7 +171,7 @@ CACHE_TYPE_V="$(normalize_arg_value "${REQUESTED_CACHE_TYPE_V:-f16}")"
 PARALLEL=1
 
 echo "effective threads: $THREADS (host=$HOST_THREADS, quota=$QUOTA_THREADS)"
-echo "effective llama profile: perf=$PERF_PROFILE single_request=1 ctx=$CTX_SIZE batch=${BATCH_SIZE:-$DEFAULT_BATCH} ubatch=${UBATCH_SIZE:-$DEFAULT_UBATCH} http_threads=$HTTP_THREADS log_verbosity=$LOG_VERBOSITY"
+echo "effective llama profile: perf=$PERF_PROFILE single_request=1 ctx=$CTX_SIZE batch=${BATCH_SIZE:-$DEFAULT_BATCH} ubatch=${UBATCH_SIZE:-$DEFAULT_UBATCH} http_threads=$HTTP_THREADS log_verbosity=$LOG_VERBOSITY mmap=$MMAP flash_attn=$FLASH_ATTN"
 
 export CACHE_TYPE_K CACHE_TYPE_V CTX_SIZE PARALLEL
 
@@ -178,7 +180,6 @@ server_args=(
   --host 0.0.0.0
   --port "${PORT:-7860}"
   --api-key "$API_PASSWORD"
-  --no-mmap
   -ngl 0
   -t "$THREADS"
   --threads-batch "${THREADS_BATCH:-$DEFAULT_THREADS_BATCH}"
@@ -188,11 +189,20 @@ server_args=(
   --ctx-size "${CTX_SIZE:-4096}"
   --cache-type-k "${CACHE_TYPE_K}"
   --cache-type-v "${CACHE_TYPE_V}"
-  --flash-attn on
   --metrics
   -lv "$LOG_VERBOSITY"
 )
 
+if [ "$MMAP" = "1" ]; then
+  server_args+=(--mmap)
+else
+  server_args+=(--no-mmap)
+fi
+
+if [ "$FLASH_ATTN" = "1" ]; then
+  server_args+=(--flash-attn on)
+fi
+
 if [ "${NO_WARMUP}" = "1" ]; then
   server_args+=(--no-warmup)
 fi