Update app.py

app.py

@@ -1,8 +1,25 @@
 import gradio as gr
 import torch
-from transformers import AutoModelForCausalLM, AutoTokenizer, pipeline
+from transformers import AutoModelForCausalLM, AutoTokenizer, pipeline,BitsAndBytesConfig
+
+#model = AutoModelForCausalLM.from_pretrained("checkpoint_500",trust_remote_code=True)
+
+model_name = "microsoft/phi-2"
+
+bnb_config = BitsAndBytesConfig(
+    load_in_4bit=True,
+    bnb_4bit_quant_type="nf4",
+    bnb_4bit_compute_dtype=torch.float16,
+)
+
+model = AutoModelForCausalLM.from_pretrained(
+    model_name,
+    quantization_config=bnb_config,
+    trust_remote_code=True
+)
+model.config.use_cache = False
+model.load_adapter("checkpoint")
 
-model = AutoModelForCausalLM.from_pretrained("checkpoint_500",trust_remote_code=True)
 tokenizer = AutoTokenizer.from_pretrained("checkpoint_500", trust_remote_code=True)
 tokenizer.pad_token = tokenizer.eos_token