add bnb config

app.py

@@ -14,9 +14,24 @@ Join us : 🌟TeamTonic🌟 is always making cool demos! Join our active builder
 """
 
 model_id = "Tonic/c4ai-command-a-03-2025-4bit_fp4"
-tokenizer = AutoTokenizer.from_pretrained(model_id, token=HF_TOKEN)
-model = AutoModelForCausalLM.from_pretrained(model_id, device_map="auto", torch_dtype=torch.bfloat16, token=HF_TOKEN)
 
+# Define quantization config for 4-bit
+quantization_config = BitsAndBytesConfig(
+    load_in_4bit=True,  # Enable 4-bit quantization
+    bnb_4bit_quant_type="fp4",  # Use FP4 quantization
+    bnb_4bit_use_double_quant=True,  # Optional: double quantization for better precision
+    llm_int8_enable_fp32_cpu_offload=True  # Allow CPU offloading for 32-bit modules
+)
+
+# Load tokenizer and model
+tokenizer = AutoTokenizer.from_pretrained(model_id, token=HF_TOKEN)
+model = AutoModelForCausalLM.from_pretrained(
+    model_id,
+    quantization_config=quantization_config,  # Apply quantization
+    device_map="auto",  # Automatically map to available devices
+    torch_dtype=torch.bfloat16,
+    token=HF_TOKEN
+)
 
 @spaces.GPU
 def generate_response(user_input, max_new_tokens, temperature):