Upload app.py

app.py

@@ -17,14 +17,14 @@ peft_model_path = "./phi2-openassistant-lora-final"
 #base_model = AutoModelForCausalLM.from_pretrained(base_model_name, quantization_config=bnb_config, device_map={"": device})
 #base_model = AutoModelForCausalLM.from_pretrained(base_model_name, torch_dtype=torch.float32, device_map={"": device})
 #base_model = AutoModelForCausalLM.from_pretrained(base_model_name, load_in_4bit=True, device_map={"": device})
-bnb_config = BitsAndBytesConfig(
-load_in_4bit=True,
-bnb_4bit_compute_dtype=torch.float16,
-bnb_4bit_quant_type="nf4",
-bnb_4bit_use_double_quant=True,
-)
-base_model = AutoModelForCausalLM.from_pretrained(base_model_name, quantization_config=bnb_config, device_map={"": device})
-
+#bnb_config = BitsAndBytesConfig(
+#load_in_4bit=True,
+#bnb_4bit_compute_dtype=torch.float16,
+#bnb_4bit_quant_type="nf4",
+#bnb_4bit_use_double_quant=True,
+#)
+#base_model = AutoModelForCausalLM.from_pretrained(base_model_name, quantization_config=bnb_config, device_map={"": device})
+base_model = AutoModelForCausalLM.from_pretrained(model_path, model_type="llama", device=device, quantization="4bit")