Update handler.py

handler.py

@@ -3,26 +3,24 @@ import torch
 from transformers import AutoTokenizer, AutoModelForCausalLM, GenerationConfig
 
 class EndpointHandler():
-    def _init_(self, path=""):
-        model_name = "deepseek-ai/deepseek-llm-67b-chat"
-        self.tokenizer = AutoTokenizer.from_pretrained(model_name)
-        self.model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=torch.bfloat16, device_map="auto")
-        self.model.generation_config = GenerationConfig.from_pretrained(model_name)
+    def __init__(self, path=""):
+        self.tokenizer = AutoTokenizer.from_pretrained(path)
+        self.model = AutoModelForCausalLM.from_pretrained(path, torch_dtype=torch.bfloat16, device_map="auto")
+        self.model.generation_config = GenerationConfig.from_pretrained(path)
         self.model.generation_config.pad_token_id = self.model.generation_config.eos_token_id
 
-    def _call_(self, data: Dict[str, Any]) -> List[Dict[str, Any]]:
-        """
-       data args:
-            inputs (:obj: `str` | `PIL.Image` | `np.array`)
-            kwargs
-      Return:
-            A :obj:`list` | `dict`: will be serialized and returned
-        """
+    def __call__(self, data: Dict[str, Any]) -> List[Dict[str, Any]]:
         inputs = data.pop('inputs', data)
         messages = [{"role": "user", "content": inputs}]
-        input_tensor = self.tokenizer.apply_chat_template(messages, add_generation_prompt=True, return_tensors="pt")
-        outputs = self.model.generate(input_tensor.to(self.model.device), max_new_tokens=100)
-        result = self.tokenizer.decode(outputs[0][input_tensor.shape[1]:], skip_special_tokens=True)
-        # pseudo
-        # self.model(input)
+
+        # Mesajları modelin anlayacağı formata dönüştürme
+        input_texts = [message["content"] for message in messages]
+        input_text = self.tokenizer.eos_token.join(input_texts)
+        input_ids = self.tokenizer.encode(input_text, return_tensors="pt")
+
+        # Modelden yanıt üretme
+        outputs = self.model.generate(input_ids.to(self.model.device), max_new_tokens=100)
+
+        # Üretilen yanıtı çözme
+        result = self.tokenizer.decode(outputs[0], skip_special_tokens=True)
         return [{"result": result}]