Update app.py

app.py

@@ -1,17 +1,24 @@
 import gradio as gr
 import torch
-from transformers import AutoModelForCausalLM, AutoTokenizer, pipeline
+from transformers import AutoModelForCausalLM, AutoTokenizer, pipeline, BitsAndBytesConfig
 
 # Model name
 MODEL_NAME = "HuggingFaceH4/zephyr-7b-beta"
 
+bnb_config = BitsAndBytesConfig(
+    load_in_4bit=True,              # quantize to 4-bit
+    bnb_4bit_compute_dtype=torch.float16,
+    bnb_4bit_use_double_quant=True,
+    bnb_4bit_quant_type="nf4",
+)
+
 # Load tokenizer and model for CPU
 tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)
 
 model = AutoModelForCausalLM.from_pretrained(
     MODEL_NAME,
-    device_map="cpu",      # ✅ force CPU
-    torch_dtype=torch.float16  # ✅ safe on CPU
+    device_map="auto",      
+    torch_dtype=torch.float16  
 )
 
 # Build pipeline