Update gigscan/server.py

gigscan/server.py

@@ -2,6 +2,7 @@ import subprocess
 import threading
 import time
 from pathlib import Path
+import os
 
 import requests
 from huggingface_hub import hf_hub_download
@@ -75,6 +76,8 @@ def start_llama_server():
     paths, download_s = timed("Model download", download_model_files)
     model_path, mmproj_path = paths
 
+    use_gpu = os.environ.get("GIGSCAN_USE_GPU", "0") == "1"
+
     cmd = [
         LLAMA_SERVER,
         "-m",
@@ -83,8 +86,6 @@ def start_llama_server():
         mmproj_path,
         "-c",
         "4096",
-        "-ngl",
-        "999",
         "--reasoning-budget",
         "0",
         "--host",
@@ -93,6 +94,12 @@ def start_llama_server():
         PORT,
     ]
 
+    if use_gpu:
+        cmd.extend(["-ngl", "999"])
+        log("GPU offload enabled")
+    else:
+        log("GPU offload disabled; running on CPU")
+
     log("Starting llama-server")
     log(" ".join(cmd))