experimental boosting changes

README.md

@@ -18,6 +18,7 @@ It automatically downloads the target Hugging Face model from `MODEL_NAME`, stor
 Set these in the Space settings:
 
 - `MODEL_NAME`: Hugging Face repo id that contains one or more GGUF files
+- `MODEL_FILE`: optional exact GGUF filename inside `MODEL_NAME`; useful for testing faster/lower quality quants
 - `HF_TOKEN`: Hugging Face token used for the download
 - `API_PASSWORD`: bearer token required by the API
 
@@ -33,6 +34,7 @@ Optional tuning variables:
 - `PERF_PROFILE`: `low_latency`, `balanced` (default), or `throughput`
 - `REASONING`: reasoning mode, accepts `True`, `False`, or `auto` and maps to llama.cpp `--reasoning`
 - `LANGSEARCH_API_KEY`: optional API key for LangSearch web search tool (free tier: 1 req/sec, 60/min, 1000/day)
+- `ENABLE_TOOLS`: set to `1` to enable llama.cpp shell tool support when `LANGSEARCH_API_KEY` is present, default `0`
 - `NO_WARMUP`: set to `1` to skip model warmup and save startup time
 - `PORT`: listen port, default `7860`
 
@@ -59,13 +61,13 @@ For small 2B models on CPU-only inference (~2-4 t/s baseline), optimize these se
 **High Impact:**
 - The server forces `--parallel 1`, so only one request runs at a time and all CPU is focused on the active query
 - `BATCH_SIZE` & `UBATCH_SIZE`: Use **64-256** for faster prompt ingestion; tiny values like 8 cause many small prompt batches
-- `CTX_SIZE`: Use **2048** unless you need long conversations; larger context increases KV/cache work
 - `PERF_PROFILE`: Use **`balanced`** for normal personal use; `throughput` only changes prompt batch sizing now
 
 **Medium Impact:**
 - `NO_WARMUP=1`: Skip model warmup to reduce startup latency by ~0.2s
-- `CACHE_TYPE_K`, `CACHE_TYPE_V`: Use `q4_0` instead of `f16` to reduce KV cache memory by 75%
-- `TOOLS`: Leave unset unless you need tool calling. Built-in shell tools add prompt/tool overhead and should not be exposed publicly.
+- `CACHE_TYPE_K`, `CACHE_TYPE_V`: Try `q4_0` instead of `f16` to reduce KV cache bandwidth and memory use at longer contexts
+- `ENABLE_TOOLS`: Leave as `0` unless you need tool calling. Built-in shell tools add prompt/tool overhead and should not be exposed publicly.
+- `MODEL_FILE`: If the repo has multiple GGUFs, test a lighter quant such as `Q4_0`, `Q4_K_S`, or `Q3_K_M` for speed/quality tradeoffs.
 
 **Example for 2B model (CPU-only):**
 ```
@@ -74,13 +76,14 @@ BATCH_SIZE=128
 UBATCH_SIZE=128
 PERF_PROFILE=balanced
 REASONING=False
+ENABLE_TOOLS=0
 ```
 
 Expected: faster first-token latency from prompt ingestion. Generation speed is still CPU-bound on the free tier, so a 2B Q4 model will usually remain in the low single-digit tokens/sec range.
 
 ## Web Search Tool (Standalone)
 
-When `LANGSEARCH_API_KEY` is set, `start.sh` enables llama.cpp's shell tool so the model can launch the local LangSearch wrapper.
+When `ENABLE_TOOLS=1` and `LANGSEARCH_API_KEY` is set, `start.sh` enables llama.cpp's shell tool so the model can launch the local LangSearch wrapper.
 
 llama.cpp does not expose arbitrary external HTTP APIs as first-class built-in tools, so LangSearch is still implemented as a local wrapper that runs through the shell tool.
 
@@ -159,7 +162,6 @@ curl http://localhost:7860/v1/completions \
   -d '{
     "model": "model",
     "prompt": "Explain KV cache in one paragraph.",
-    "max_tokens": 128,
     "stream": false
   }'
 ```

download_model.py

@@ -54,24 +54,26 @@ def main() -> None:
     models_dir.mkdir(parents=True, exist_ok=True)
 
     repo_marker = models_dir / "model.repo"
+    model_file_override = os.environ.get("MODEL_FILE", "").strip()
+    selection_key = f"file:{model_file_override}" if model_file_override else ""
+
     # If a repo marker exists, check whether the previously downloaded file exists
     if repo_marker.exists():
         marker = repo_marker.read_text(encoding="utf-8").strip()
-        if "|" in marker:
-            prev_repo, prev_file = marker.split("|", 1)
-        else:
-            prev_repo, prev_file = marker, ""
-        if prev_repo == model_name and prev_file:
+        marker_parts = marker.split("|")
+        prev_repo = marker_parts[0] if len(marker_parts) > 0 else ""
+        prev_file = marker_parts[1] if len(marker_parts) > 1 else ""
+        prev_selection_key = marker_parts[2] if len(marker_parts) > 2 else ""
+        if prev_repo == model_name and prev_file and (not selection_key or prev_selection_key == selection_key):
             cached = models_dir / prev_file
             if cached.exists():
                 print(f"using cached model: {cached}")
                 return
+        elif prev_repo == model_name and prev_file and selection_key and prev_selection_key != selection_key:
+            print("cached model does not match MODEL_FILE override; selecting requested file", flush=True)
     # Use a temporary directory (under /tmp) for downloads to avoid permission issues
     download_dir = Path(tempfile.mkdtemp(prefix="hf-download-"))
 
-    # allow user to request a specific file in the repo (useful when repo has no .gguf)
-    model_file_override = os.environ.get("MODEL_FILE", "").strip()
-
     # previous cache check moved above
 
     # Ensure any stale temp dir is removed with a permissive onerror handler
@@ -161,7 +163,7 @@ def main() -> None:
 
     if repo_marker.exists():
         repo_marker.unlink()
-    repo_marker.write_text(f"{model_name}|{selected_basename}", encoding="utf-8")
+    repo_marker.write_text(f"{model_name}|{selected_basename}|{selection_key}", encoding="utf-8")
 
     try:
         shutil.rmtree(download_dir, onerror=_rmtree_onerror)
@@ -171,4 +173,4 @@ def main() -> None:
 
 
 if __name__ == "__main__":
-    main()
+    main()

start.sh

@@ -75,6 +75,7 @@ REQUESTED_CACHE_TYPE_K="${CACHE_TYPE_K:-}"
 REQUESTED_CACHE_TYPE_V="${CACHE_TYPE_V:-}"
 export REASONING="${REASONING:-auto}"
 export TOOLS="${TOOLS:-}"
+export ENABLE_TOOLS="${ENABLE_TOOLS:-0}"
 export NO_WARMUP="${NO_WARMUP:-0}"
 export LANGSEARCH_API_KEY="${LANGSEARCH_API_KEY:-}"
 
@@ -192,7 +193,7 @@ case "${REASONING,,}" in
     ;;
 esac
 
-if [ -n "$LANGSEARCH_API_KEY" ] && [ -z "$TOOLS" ]; then
+if [ "$ENABLE_TOOLS" = "1" ] && [ -n "$LANGSEARCH_API_KEY" ] && [ -z "$TOOLS" ]; then
   TOOLS="exec_shell_command"
 fi