Create run_inference.py

run_inference.py

@@ -0,0 +1,31 @@
+from peft import AutoPeftModelForCausalLM
+from transformers import AutoTokenizer
+import re
+
+
+PROMPT = "YOUR PROMPT HERE"
+MAX_LENGTH = 32768  # Do not change
+DEVICE = "cuda"
+
+
+model_id = "agarkovv/Ministral-8B-Instruct-2410-LoRA-trading"
+base_model_id = "mistralai/Ministral-8B-Instruct-2410"
+
+model = AutoPeftModelForCausalLM.from_pretrained(model_id)
+tokenizer = AutoTokenizer.from_pretrained(base_model_id)
+
+model = model.to(DEVICE)
+model.eval()
+inputs = tokenizer(
+    PROMPT, return_tensors="pt", padding=False, max_length=MAX_LENGTH, truncation=True
+)
+inputs = {key: value.to(model.device) for key, value in inputs.items()}
+
+res = model.generate(
+    **inputs,
+    use_cache=True,
+    max_new_tokens=MAX_LENGTH,
+)
+output = tokenizer.decode(res[0], skip_special_tokens=True)
+answer = re.sub(r".*\[/INST\]\s*", "", output, flags=re.DOTALL)
+print(answer)