Update app.py

app.py

@@ -6,11 +6,23 @@ from typing import List, Dict, Any, Optional
 from functools import lru_cache
 from xml.etree import ElementTree as ET
 from transformers import AutoTokenizer, AutoModelForCausalLM
+from transformers import AutoTokenizer, AutoModelForCausalLM
 try:
-    from transformers import BitsAndBytesConfig
-except ImportError:
+    from transformers import BitsAndBytesConfig  # exists even if bitsandbytes isn't installed
+except Exception:
     BitsAndBytesConfig = None
 
+# Normalize QUANTIZE env
+QUANTIZE = os.environ.get("QUANTIZE", "none").strip().lower()
+
+# Detect bitsandbytes presence
+try:
+    import bitsandbytes as _bnb  # noqa: F401
+    _BNB_AVAILABLE = True
+except Exception:
+    _BNB_AVAILABLE = False
+
+
 
 import numpy as np
 import requests
@@ -205,7 +217,9 @@ if ADAPTER_REPO:
 dlog("LLM", f"Loading base model: {BASE_MODEL}")
 tokenizer_lm = AutoTokenizer.from_pretrained(BASE_MODEL, use_fast=False)
 
-if QUANTIZE in {"8bit", "4bit"} and BitsAndBytesConfig is not None:
+use_bnb = QUANTIZE in {"8bit", "4bit"} and BitsAndBytesConfig is not None and _BNB_AVAILABLE
+
+if use_bnb:
     bnb_config = BitsAndBytesConfig(
         load_in_8bit=(QUANTIZE == "8bit"),
         load_in_4bit=(QUANTIZE == "4bit"),
@@ -219,7 +233,7 @@ if QUANTIZE in {"8bit", "4bit"} and BitsAndBytesConfig is not None:
         quantization_config=bnb_config,
     )
 else:
-    # Fallback / default: fp16 (no bitsandbytes required)
+    # Default / fallback: fp16 (no bitsandbytes required)
     base_model = AutoModelForCausalLM.from_pretrained(
         BASE_MODEL,
         torch_dtype=dtype,
@@ -228,6 +242,7 @@ else:
 
 
 
+
 dlog("LLM", f"Loading LoRA adapter from: {ADAPTER_PATH}")
 model_lm = PeftModel.from_pretrained(base_model, ADAPTER_PATH)
 model_lm.eval()