Upload app.py

app.py

@@ -3,7 +3,7 @@ import os
 import gradio as gr
 import torch
 from PIL import Image
-from transformers import AutoTokenizer
+from transformers import AutoTokenizer, AutoModelForCausalLM
 import timm
 from torchvision import transforms
 from llama_cpp import Llama
@@ -62,17 +62,21 @@ image_encoder.eval() # Set to evaluation mode
 #)
 
 
-base_model = Llama.from_pretrained(
-	repo_id="QuantFactory/Phi-3-mini-4k-instruct-GGUF",
-	#filename="Phi-3-mini-4k-instruct.Q2_K.gguf",
-    filename="Phi-3-mini-4k-instruct.Q4_K_M.gguf",
-    n_gpu_layers=0,
-    n_ctx=2048,
-    verbose=True 
-)
+#base_model = Llama.from_pretrained(
+#	repo_id="QuantFactory/Phi-3-mini-4k-instruct-GGUF",
+#	filename="Phi-3-mini-4k-instruct.Q2_K.gguf",
+#    n_gpu_layers=0,
+#    n_ctx=2048,
+#    verbose=True 
+#)
+
+base_model_name="microsoft/Phi-3-mini-4k-instruct"
+device = "cpu"
 
+base_model = AutoModelForCausalLM.from_pretrained(base_model_name, torch_dtype=torch.float32, device_map={"": device})
 
 
+# Load and merge
 model = PeftModel.from_pretrained(base_model, peft_model_path, offload_dir='./offload')
 model = model.merge_and_unload()
 print("phi-3 model loaded sucessfully")