Create handle.py

handle.py

@@ -0,0 +1,30 @@
+from unsloth import FastLanguageModel
+import torch
+
+class EndpointHandler:
+    def __init__(self, path=""):
+        self.model, self.tokenizer = FastLanguageModel.from_pretrained(
+            model_name=path,
+            max_seq_length=2048,
+            dtype=None,
+            load_in_4bit=True,
+        )
+        FastLanguageModel.for_inference(self.model)
+
+    def __call__(self, data: dict):
+        inputs_text = data.pop("inputs", "")
+        parameters = data.pop("parameters", {})
+
+        inputs = self.tokenizer(
+            inputs_text,
+            return_tensors="pt"
+        ).to("cuda")
+
+        outputs = self.model.generate(
+            **inputs,
+            max_new_tokens=parameters.get("max_new_tokens", 256),
+            temperature=parameters.get("temperature", 0.7),
+            do_sample=True,
+        )
+
+        return self.tokenizer.decode(outputs[0], skip_special_tokens=True)