Update app.py

app.py

@@ -1,6 +1,6 @@
 import torch
 import gradio as gr
-from transformers import AutoTokenizer, AutoModelForCausalLM, BitsAndBytesConfig, TextIteratorStreamer
+from transformers import AutoTokenizer, AutoModelForCausalLM, TextIteratorStreamer
 from threading import Thread
 import sys
 
@@ -16,35 +16,44 @@ You are not a tool. You are SAMAI — think, then speak."""
 print("Loading model...")
 
 # ===== Check for CUDA =====
-if not torch.cuda.is_available():
-    print("❌ No NVIDIA GPU detected! This model requires a GPU to run.")
-    print("   Please select a GPU hardware in your Hugging Face Space settings")
-    print("   (e.g., T4 small) or run locally with a CUDA-compatible GPU.")
-    sys.exit(1)   # stop here – no point continuing
-
-# ===== GPU path – use 4-bit with optimisations =====
-bnb_config = BitsAndBytesConfig(
-    load_in_4bit=True,
-    bnb_4bit_quant_type="nf4",
-    bnb_4bit_compute_dtype=torch.float16,   # T4 native
-    bnb_4bit_use_double_quant=True,
-)
+has_cuda = torch.cuda.is_available()
+print(f"CUDA available: {has_cuda}")
 
 tokenizer = AutoTokenizer.from_pretrained(MODEL_ID, trust_remote_code=True)
 
-model = AutoModelForCausalLM.from_pretrained(
-    MODEL_ID,
-    quantization_config=bnb_config,
-    device_map="cuda",                       # force GPU
-    trust_remote_code=True,
-    dtype=torch.float16,                     # base weights in FP16
-    attn_implementation="sdpa",              # memory‑efficient attention
-)
+if has_cuda:
+    # ----- GPU path: use 4-bit quantization -----
+    from transformers import BitsAndBytesConfig
+    bnb_config = BitsAndBytesConfig(
+        load_in_4bit=True,
+        bnb_4bit_quant_type="nf4",
+        bnb_4bit_compute_dtype=torch.float16,
+        bnb_4bit_use_double_quant=True,
+    )
+    model = AutoModelForCausalLM.from_pretrained(
+        MODEL_ID,
+        quantization_config=bnb_config,
+        device_map="cuda",
+        trust_remote_code=True,
+        dtype=torch.float16,
+        attn_implementation="sdpa",
+    )
+else:
+    # ----- CPU path: load full FP16 model (requires ~28 GB RAM) -----
+    print("⚠️ No GPU detected. Loading the model in FP16 on CPU (memory intensive).")
+    print("   If you run out of memory, consider using a smaller model or a GPU.")
+    model = AutoModelForCausalLM.from_pretrained(
+        MODEL_ID,
+        device_map="cpu",
+        trust_remote_code=True,
+        torch_dtype=torch.float16,          # half precision to save memory
+        low_cpu_mem_usage=True,             # memory efficient loading
+    )
 
 model.eval()
 print("✅ Ready!")
 
-# ===== Generation function (tuple history – works with older Gradio) =====
+# ===== Generation function (tuple history) =====
 def respond(message, history, max_tokens, temperature):
     messages = [{"role": "system", "content": SAMAI_SYSTEM}]
     for user_msg, bot_msg in history: