add load in 4bit

app.py

@@ -1,7 +1,7 @@
 import gradio as gr
 # from huggingface_hub import InferenceClient
 from peft import AutoPeftModelForCausalLM
-from transformers import AutoTokenizer, TextStreamer
+from transformers import AutoTokenizer, TextStreamer, BitsAndBytesConfig
 
 """
 For more information on `huggingface_hub` Inference API support, please check the docs: https://huggingface.co/docs/huggingface_hub/v0.22.2/en/guides/inference
@@ -9,8 +9,12 @@ For more information on `huggingface_hub` Inference API support, please check th
 # client = InferenceClient("EITD/lora_model", token=os.getenv("HF_TOKEN"))
 
 model_name = "EITD/lora_model"
+quantization_config = BitsAndBytesConfig(
+    load_in_4bit=True
+)
 model = AutoPeftModelForCausalLM.from_pretrained(
-    model_name
+    model_name,
+    quantization_config=quantization_config
 )
 tokenizer = AutoTokenizer.from_pretrained(model_name)