Update app.py

app.py

@@ -26,7 +26,8 @@ def load_model():
         bnb_4bit_use_double_quant=True,
         bnb_4bit_quant_type="nf4",
         # bfloat16 is recommended for Llama models on modern GPUs (A100, V100, T4)
-        bnb_4bit_compute_dtype=torch.bfloat16 
+        bnb_4bit_compute_dtype=torch.bfloat16,
+        llm_int8_enable_fp32_cpu_offload=True 
     )
 
     # Load the base model (ensure trust_remote_code=True for custom Llama models)