Upload app.py

app.py

@@ -24,8 +24,8 @@ peft_model_path = "./phi2-openassistant-lora-final"
 #bnb_4bit_use_double_quant=True,
 #)
 #base_model = AutoModelForCausalLM.from_pretrained(base_model_name, quantization_config=bnb_config, device_map={"": device})
-base_model = AutoModelForCausalLM.from_pretrained(base_model_name, model_type="llama", device_map={"": device}, quantization="4bit")
-
+#base_model = AutoModelForCausalLM.from_pretrained(base_model_name, model_type="llama", device_map={"": device}, quantization="4bit")
+base_model = AutoModelForCausalLM.from_pretrained(base_model_name, load_in_4bit=True, device_map={"": device}, torch_dtype=torch.float16)