handler.py

handler.py

@@ -0,0 +1,28 @@
+from typing import Dict, List, Any
+import torch
+from transformers import AutoTokenizer, AutoModelForCausalLM, GenerationConfig
+
+class EndpointHandler():
+    def _init_(self, path=""):
+        model_name = "deepseek-ai/deepseek-llm-67b-chat"
+        self.tokenizer = AutoTokenizer.from_pretrained(model_name)
+        self.model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=torch.bfloat16, device_map="auto")
+        self.model.generation_config = GenerationConfig.from_pretrained(model_name)
+        self.model.generation_config.pad_token_id = self.model.generation_config.eos_token_id
+
+    def _call_(self, data: Dict[str, Any]) -> List[Dict[str, Any]]:
+        """
+       data args:
+            inputs (:obj: `str` | `PIL.Image` | `np.array`)
+            kwargs
+      Return:
+            A :obj:`list` | `dict`: will be serialized and returned
+        """
+        inputs = data.pop('inputs', data)
+        messages = [{"role": "user", "content": inputs}]
+        input_tensor = self.tokenizer.apply_chat_template(messages, add_generation_prompt=True, return_tensors="pt")
+        outputs = self.model.generate(input_tensor.to(self.model.device), max_new_tokens=100)
+        result = self.tokenizer.decode(outputs[0][input_tensor.shape[1]:], skip_special_tokens=True)
+        # pseudo
+        # self.model(input)
+        return [{"result": result}]