a

app.py

@@ -5,6 +5,7 @@ from huggingface_hub import InferenceClient
 Copied from inference in colab notebook
 """
 
+from optimum.onnxruntime import ORTModelForCausalLM
 from transformers import LlamaForCausalLM, AutoTokenizer
 import torch
 
@@ -15,11 +16,7 @@ model_path = "llama_lora_model_1"
 tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, legacy=False)
 
 # Load model
-model = LlamaForCausalLM.from_pretrained(
-    model_path,
-    torch_dtype=torch.float32,  # Adjust based on your environment
-    device_map="cpu"  # Use CPU for inference
-)
+model = ORTModelForCausalLM.from_pretrained(model_path, from_transformers=True)
 
 # Define the response function
 def respond(

requirements.txt

@@ -2,4 +2,5 @@ huggingface_hub==0.25.2
 
 unsloth
 transformers
-accelerate
+accelerate
+optimum[onnxruntime]