Update app.py

app.py

@@ -11,13 +11,13 @@ CORS(app)
 # Model (UNGATED, FREE)
 # ----------------------------
 MODEL_NAME = "microsoft/phi-3-mini-4k-instruct"
-DEVICE = "cpu"
+DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
 
+tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)
 model = AutoModelForCausalLM.from_pretrained(
     MODEL_NAME,
-    torch_dtype=torch.float32,
-
-model.to(DEVICE))
+    torch_dtype=torch.float16 if DEVICE == "cuda" else torch.float32
+).to(DEVICE)
 
 # ----------------------------
 # SYSTEM PROMPT
@@ -57,4 +57,4 @@ def ask():
 # START SERVER
 # ----------------------------
 if __name__ == "__main__":
-    app.run(host="0.0.0.0", port=7860)
+    app.run(host="0.0.0.0", port=7860)