Upload 2 files

app.py

@@ -39,6 +39,7 @@ N_CTX         = int(os.environ.get("N_CTX",     "4096"))
 N_THREADS     = int(os.environ.get("N_THREADS", str(os.cpu_count() or 4)))
 N_BATCH       = int(os.environ.get("N_BATCH",   "512"))
 VERBOSE       = os.environ.get("VERBOSE",       "false").lower() == "true"
+HF_TOKEN      = os.environ.get("HF_TOKEN",      "")
 
 # ---------------------------------------------------------------------------
 # Lazy model holder
@@ -57,27 +58,33 @@ def _download_model() -> None:
         return
 
     path.parent.mkdir(parents=True, exist_ok=True)
-    logger.info(f"Model not found — downloading from {MODEL_URL} …")
+    logger.info(f"Model not found — downloading from {MODEL_URL} ...")
     logger.info("This will take a while on first boot (file is ~9 GB).")
 
     import urllib.request
 
+    headers = {"Authorization": f"Bearer {HF_TOKEN}"} if HF_TOKEN else {}
+    if not HF_TOKEN:
+        logger.warning("HF_TOKEN not set — download may fail for gated models.")
+
     tmp = Path(str(MODEL_PATH) + ".part")
-    last_pct = -1
-
-    def _progress(block_num, block_size, total_size):
-        nonlocal last_pct
-        if total_size <= 0:
-            return
-        pct = int(block_num * block_size * 100 / total_size)
-        pct = min(pct, 100)
-        if pct != last_pct and pct % 5 == 0:
-            logger.info(f"Download progress: {pct}%")
-            last_pct = pct
-
-    urllib.request.urlretrieve(MODEL_URL, tmp, reporthook=_progress)
+    req = urllib.request.Request(MODEL_URL, headers=headers)
+
+    with urllib.request.urlopen(req) as response, open(tmp, "wb") as out:
+        total = int(response.headers.get("Content-Length", 0))
+        downloaded = 0
+        last_pct = -1
+        while chunk := response.read(1 << 20):  # 1 MB chunks
+            out.write(chunk)
+            downloaded += len(chunk)
+            if total:
+                pct = min(int(downloaded * 100 / total), 100)
+                if pct != last_pct and pct % 5 == 0:
+                    logger.info(f"Download progress: {pct}%")
+                    last_pct = pct
+
     tmp.rename(path)
-    logger.info(f"Download complete → {MODEL_PATH}")
+    logger.info(f"Download complete -> {MODEL_PATH}")
 
 
 def _load_model_background() -> None:
@@ -85,7 +92,7 @@ def _load_model_background() -> None:
     global _llm, _llm_error
     try:
         _download_model()
-        logger.info(f"Loading model into memory from {MODEL_PATH} …")
+        logger.info(f"Loading model into memory from {MODEL_PATH} ...")
         llm = Llama(
             model_path=MODEL_PATH,
             n_ctx=N_CTX,
@@ -97,7 +104,7 @@ def _load_model_background() -> None:
         )
         with _llm_lock:
             _llm = llm
-        logger.info("Model loaded and ready ✓")
+        logger.info("Model loaded and ready")
     except Exception as exc:
         _llm_error = str(exc)
         logger.error(f"Failed to load model: {exc}")
@@ -140,7 +147,7 @@ async def startup_event():
     starts immediately and stays responsive during the (long) load phase."""
     t = threading.Thread(target=_load_model_background, daemon=True)
     t.start()
-    logger.info("Server is up. Model loading in background — see /health for status.")
+    logger.info("Server is up. Model loading in background -- see /health for status.")
 
 
 # ---------------------------------------------------------------------------

docker-compose.yml

@@ -21,6 +21,7 @@ services:
       N_THREADS: "8"   # set to your physical CPU core count
       N_BATCH: "512"
       VERBOSE: "false"
+      HF_TOKEN: "hf_xxxxxxxxxxxxxxxxxxxx"   # replace with your HuggingFace read token
     restart: unless-stopped
 
 volumes: