Update app.py

app.py

@@ -1,13 +1,18 @@
 import gradio as gr
-from transformers import AutoTokenizer, AutoModelForCausalLM, GenerationConfig
+from transformers import AutoTokenizer, AutoModelForCausalLM, GenerationConfig,BitsAndBytesConfig
 
 import torch
 
 # Check if a GPU is available
 device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 
+bnb_config = BitsAndBytesConfig(
+        load_in_4bit=True, bnb_4bit_quant_type="nf4", bnb_4bit_compute_dtype="float16", bnb_4bit_use_double_quant=True
+    )
+
+
 tokenizer = AutoTokenizer.from_pretrained("truongghieu/deci-finetuned", trust_remote_code=True)
-model = AutoModelForCausalLM.from_pretrained("truongghieu/deci-finetuned", trust_remote_code=True,low_cpu_mem_usage=True)
+model = AutoModelForCausalLM.from_pretrained("truongghieu/deci-finetuned", trust_remote_code=True, quantization_config=bnb_config)
 
 # Move the model to the GPU if available
 model.to(device)
@@ -22,6 +27,8 @@ generation_config = GenerationConfig(
     pad_token_id=tokenizer.eos_token_id
 )
 
+
+
 # Define a function that takes a text input and generates a text output
 def generate_text(text):
     input_text = text