Update app.py

app.py

@@ -1,5 +1,6 @@
 import gradio as gr
-from transformers import AutoModelForCausalLM, AutoTokenizer, pipeline
+from transformers import AutoModelForCausalLM, AutoTokenizer, pipeline, BitsAndBytesConfig
+
 from peft import PeftModel
 import torch
 import os 
@@ -21,7 +22,14 @@ base_model = AutoModelForCausalLM.from_pretrained(
     token=API_KEY
 )
 
-model = PeftModel.from_pretrained(base_model, PEFT_ADAPTER, token=API_KEY, load_in_8bit=True)
+nf4_config = BitsAndBytesConfig(
+   load_in_4bit=True,
+   bnb_4bit_quant_type="nf4",
+   bnb_4bit_use_double_quant=True,
+   bnb_4bit_compute_dtype=torch.bfloat16
+)
+
+model = PeftModel.from_pretrained(base_model, PEFT_ADAPTER, token=API_KEY, quantization_config=nf4_config)
 
 pipe = pipeline(
     "text-generation",