Upload app.py

app.py

@@ -15,7 +15,8 @@ peft_model_path = "./phi2-openassistant-lora-final"
 # Load the base model with 4-bit quantization
 #bnb_config = BitsAndBytesConfig(load_in_4bit=True)  # Ensure compatibility
 #base_model = AutoModelForCausalLM.from_pretrained(base_model_name, quantization_config=bnb_config, device_map={"": device})
-base_model = AutoModelForCausalLM.from_pretrained(base_model_name, torch_dtype=torch.float32, device_map={"": device})
+#base_model = AutoModelForCausalLM.from_pretrained(base_model_name, torch_dtype=torch.float32, device_map={"": device})
+base_model = AutoModelForCausalLM.from_pretrained(base_model_name, load_in_4bit=True,, device_map={"": device})
 
 # Load LoRA adapter
 model = PeftModel.from_pretrained(base_model, peft_model_path)
@@ -23,6 +24,7 @@ model = model.merge_and_unload()  # Merge LoRA with base model
 
 # Load tokenizer
 tokenizer = AutoTokenizer.from_pretrained(base_model_name)
+tokenizer.pad_token = tokenizer.eos_token
 
 # Define prediction function
 def generate_response(prompt):