remove parallel support for faster inference

README.md

@@ -24,17 +24,16 @@ Set these in the Space settings:
 Optional tuning variables:
 
 - `CTX_SIZE`: context size, default `4096` (reduce to 2048-4096 for small models to improve speed)
-- `THREADS`: CPU thread count, default auto-detect via nproc
+- `THREADS`: CPU thread count, default auto-detect via cgroup CPU quota, then `nproc`
 - `THREADS_BATCH`: batch thread count, default matches THREADS
-- `BATCH_SIZE`: prompt batch size, default 8 (increase to 16-32 for better throughput)
-- `UBATCH_SIZE`: micro-batch size, default 8 (increase to 16-32 for better throughput)
+- `BATCH_SIZE`: prompt batch size, default 128 for balanced mode
+- `UBATCH_SIZE`: micro-batch size, default 128 for balanced mode
 - `CACHE_TYPE_K`: KV cache type for keys, default `f16` (use `q4_0` for memory savings)
 - `CACHE_TYPE_V`: KV cache type for values, default `f16` (use `q4_0` for memory savings)
 - `PERF_PROFILE`: `low_latency`, `balanced` (default), or `throughput`
 - `REASONING`: reasoning mode, accepts `True`, `False`, or `auto` and maps to llama.cpp `--reasoning`
 - `LANGSEARCH_API_KEY`: optional API key for LangSearch web search tool (free tier: 1 req/sec, 60/min, 1000/day)
 - `NO_WARMUP`: set to `1` to skip model warmup and save startup time
-- `PARALLEL`: optional override for concurrent prompt decode slots
 - `PORT`: listen port, default `7860`
 
 ## Endpoints
@@ -55,27 +54,29 @@ Model downloads and HF cache live on the `/data` bucket so restarts do not redow
 
 ## Performance Optimization
 
-For small 2B models on CPU-only inference (~2-3 t/s baseline), optimize these settings:
+For small 2B models on CPU-only inference (~2-4 t/s baseline), optimize these settings:
 
 **High Impact:**
-- `CTX_SIZE`: Reduce from 32000 to **2048** or **4096** (KV cache memory dominates)
-- `BATCH_SIZE` & `UBATCH_SIZE`: Increase to **16-32** for better CPU utilization
-- `PERF_PROFILE`: Use **`throughput`** instead of balanced for batch mode
+- The server forces `--parallel 1`, so only one request runs at a time and all CPU is focused on the active query
+- `BATCH_SIZE` & `UBATCH_SIZE`: Use **64-256** for faster prompt ingestion; tiny values like 8 cause many small prompt batches
+- `CTX_SIZE`: Use **2048** unless you need long conversations; larger context increases KV/cache work
+- `PERF_PROFILE`: Use **`balanced`** for normal personal use; `throughput` only changes prompt batch sizing now
 
 **Medium Impact:**
 - `NO_WARMUP=1`: Skip model warmup to reduce startup latency by ~0.2s
 - `CACHE_TYPE_K`, `CACHE_TYPE_V`: Use `q4_0` instead of `f16` to reduce KV cache memory by 75%
+- `TOOLS`: Leave unset unless you need tool calling. Built-in shell tools add prompt/tool overhead and should not be exposed publicly.
 
 **Example for 2B model (CPU-only):**
 ```
 CTX_SIZE=2048
-BATCH_SIZE=16
-UBATCH_SIZE=16
-PERF_PROFILE=throughput
-NO_WARMUP=1
+BATCH_SIZE=128
+UBATCH_SIZE=128
+PERF_PROFILE=balanced
+REASONING=False
 ```
 
-Expected: 3-5 t/s generation speed (vs 2-3 t/s with default settings).
+Expected: faster first-token latency from prompt ingestion. Generation speed is still CPU-bound on the free tier, so a 2B Q4 model will usually remain in the low single-digit tokens/sec range.
 
 ## Web Search Tool (Standalone)
 

start.sh

@@ -1,7 +1,65 @@
 #!/usr/bin/env bash
 set -euo pipefail
 
-THREADS="${THREADS:-$(nproc)}"
+detect_cpu_quota_threads() {
+  local quota period threads
+
+  if [ -r /sys/fs/cgroup/cpu.max ]; then
+    read -r quota period < /sys/fs/cgroup/cpu.max || true
+    if [ "${quota:-max}" != "max" ] && [ "${period:-0}" -gt 0 ] 2>/dev/null; then
+      threads=$(( (quota + period - 1) / period ))
+      [ "$threads" -gt 0 ] && echo "$threads" && return
+    fi
+  fi
+
+  if [ -r /sys/fs/cgroup/cpu/cpu.cfs_quota_us ] && [ -r /sys/fs/cgroup/cpu/cpu.cfs_period_us ]; then
+    quota=$(cat /sys/fs/cgroup/cpu/cpu.cfs_quota_us)
+    period=$(cat /sys/fs/cgroup/cpu/cpu.cfs_period_us)
+    if [ "${quota:-0}" -gt 0 ] && [ "${period:-0}" -gt 0 ] 2>/dev/null; then
+      threads=$(( (quota + period - 1) / period ))
+      [ "$threads" -gt 0 ] && echo "$threads" && return
+    fi
+  fi
+
+  echo 0
+}
+
+detect_memory_kb() {
+  local value
+
+  if [ -r /sys/fs/cgroup/memory.max ]; then
+    value=$(cat /sys/fs/cgroup/memory.max)
+    if [ "$value" != "max" ] && [ "${value:-0}" -gt 0 ] 2>/dev/null; then
+      echo $(( value / 1024 ))
+      return
+    fi
+  fi
+
+  if [ -r /sys/fs/cgroup/memory/memory.limit_in_bytes ]; then
+    value=$(cat /sys/fs/cgroup/memory/memory.limit_in_bytes)
+    if [ "${value:-0}" -gt 0 ] && [ "$value" -lt 9223372036854771712 ] 2>/dev/null; then
+      echo $(( value / 1024 ))
+      return
+    fi
+  fi
+
+  if [ -r /proc/meminfo ]; then
+    awk '/MemTotal/ {print $2}' /proc/meminfo || echo 0
+  else
+    echo 0
+  fi
+}
+
+HOST_THREADS="$(nproc)"
+QUOTA_THREADS="$(detect_cpu_quota_threads)"
+if [ -z "${THREADS:-}" ]; then
+  if [ "$QUOTA_THREADS" -gt 0 ] && [ "$QUOTA_THREADS" -lt "$HOST_THREADS" ]; then
+    THREADS="$QUOTA_THREADS"
+  else
+    THREADS="$HOST_THREADS"
+  fi
+fi
+
 export OMP_NUM_THREADS="$THREADS"
 export OPENBLAS_NUM_THREADS="$THREADS"
 export MKL_NUM_THREADS="$THREADS"
@@ -10,11 +68,11 @@ export NUMEXPR_NUM_THREADS="$THREADS"
 export HF_HOME="${HF_HOME:-/data/hf-cache}"
 export MODEL_DIR="${MODEL_DIR:-/data/models}"
 export MODEL_PATH="${MODEL_PATH:-/data/models/model.gguf}"
-export CTX_SIZE="${CONTEXT_LENGTH:-${CTX_SIZE:-4096}}"
+REQUESTED_CTX_SIZE="${CONTEXT_LENGTH:-${CTX_SIZE:-}}"
 export QUANT_PREFERENCE="${QUANT_PREFERENCE:-q4_k_m,q4_k_s,q4_k,q4_0,q4_1}"
 export PERF_PROFILE="${PERF_PROFILE:-balanced}"
-export CACHE_TYPE_K="${CACHE_TYPE_K:-f16}"
-export CACHE_TYPE_V="${CACHE_TYPE_V:-f16}"
+REQUESTED_CACHE_TYPE_K="${CACHE_TYPE_K:-}"
+REQUESTED_CACHE_TYPE_V="${CACHE_TYPE_V:-}"
 export REASONING="${REASONING:-auto}"
 export TOOLS="${TOOLS:-}"
 export NO_WARMUP="${NO_WARMUP:-0}"
@@ -55,13 +113,13 @@ LLAMA_SERVER_BIN="${LLAMA_SERVER_BIN:-/usr/local/bin/llama-server}"
 # set server profile defaults
 case "$PERF_PROFILE" in
   low_latency)
-    DEFAULT_BATCH=1
-    DEFAULT_UBATCH=1
-    DEFAULT_THREADS_BATCH=1
+    DEFAULT_BATCH=64
+    DEFAULT_UBATCH=64
+    DEFAULT_THREADS_BATCH=${THREADS}
     ;;
   balanced)
-    DEFAULT_BATCH=8
-    DEFAULT_UBATCH=8
+    DEFAULT_BATCH=128
+    DEFAULT_UBATCH=128
     DEFAULT_THREADS_BATCH=${THREADS}
     ;;
   throughput)
@@ -70,37 +128,32 @@ case "$PERF_PROFILE" in
     DEFAULT_THREADS_BATCH=${THREADS}
     ;;
   *)
-    DEFAULT_BATCH=8
-    DEFAULT_UBATCH=8
+    DEFAULT_BATCH=128
+    DEFAULT_UBATCH=128
     DEFAULT_THREADS_BATCH=${THREADS}
     ;;
 esac
 
-# Adjust defaults for low-memory environments (Spaces free tier: ~16GB)
-if [ -r /proc/meminfo ]; then
-  MEM_KB=$(awk '/MemTotal/ {print $2}' /proc/meminfo || echo 0)
-else
-  MEM_KB=0
-fi
-echo "detected memory (KB): $MEM_KB"
+# Adjust defaults for low-memory environments (Spaces free tier: ~16GB).
+# Use cgroup limits first because /proc/meminfo can report the host machine.
+MEM_KB="$(detect_memory_kb)"
+echo "detected effective memory (KB): $MEM_KB"
 if [ "$MEM_KB" -gt 0 ] && [ "$MEM_KB" -le 18000000 ]; then
-  echo "low-memory profile detected: adjusting ctx-size and cache types"
-  # reduce default context to save KV cache memory
-  CTX_SIZE="${CTX_SIZE:-2048}"
-  # use compact cache formats
-  CACHE_TYPE_K="${CACHE_TYPE_K:-f16}"
-  CACHE_TYPE_V="${CACHE_TYPE_V:-f16}"
-  # further reduce batch sizes if balanced/throughput
-  if [ "$PERF_PROFILE" = "balanced" ]; then
-    DEFAULT_BATCH=4
-    DEFAULT_UBATCH=4
-  elif [ "$PERF_PROFILE" = "throughput" ]; then
-    DEFAULT_BATCH=64
-    DEFAULT_UBATCH=64
-  fi
+  echo "low-memory profile detected: using compact defaults"
+  DEFAULT_CTX_SIZE=2048
+else
+  DEFAULT_CTX_SIZE=4096
 fi
 
-export CACHE_TYPE_K CACHE_TYPE_V CTX_SIZE
+CTX_SIZE="${REQUESTED_CTX_SIZE:-$DEFAULT_CTX_SIZE}"
+CACHE_TYPE_K="${REQUESTED_CACHE_TYPE_K:-f16}"
+CACHE_TYPE_V="${REQUESTED_CACHE_TYPE_V:-f16}"
+PARALLEL=1
+
+echo "effective threads: $THREADS (host=$HOST_THREADS, quota=$QUOTA_THREADS)"
+echo "effective llama profile: perf=$PERF_PROFILE single_request=1 ctx=$CTX_SIZE batch=${BATCH_SIZE:-$DEFAULT_BATCH} ubatch=${UBATCH_SIZE:-$DEFAULT_UBATCH}"
+
+export CACHE_TYPE_K CACHE_TYPE_V CTX_SIZE PARALLEL
 
 server_args=(
   --model "$MODEL_PATH"
@@ -148,11 +201,9 @@ if [[ -n "${TOOLS}" ]]; then
   echo "enabled tools: $TOOLS"
 fi
 
-if [[ -n "${PARALLEL:-}" ]]; then
-  server_args+=(--parallel "$PARALLEL")
-fi
+server_args+=(--parallel "$PARALLEL")
 
 server_cmd=("$LLAMA_SERVER_BIN" "${server_args[@]}")
 echo "server command: $LLAMA_SERVER_BIN ${server_args[*]}"
 echo "starting llama-server"
-exec su -s /bin/sh appuser -c "$(printf '%q ' "${server_cmd[@]}")"
+exec su -s /bin/sh appuser -c "$(printf '%q ' "${server_cmd[@]}")"