Added model inference code

handler.py

@@ -21,6 +21,10 @@ class EndpointHandler():
                 quantization_config=None,
                 torch_dtype=torch.float,     # data type is float
                 device_map="auto")
+         self.tokenizer = AutoTokenizer.from_pretrained(path, trust_remote_code=True)
+         self.tokenizer.padding_side = "left"
+         self.tokenizer.pad_token = self.tokenizer.eos_token
+         self.tokenizer.add_eos_token = True
 
     def __call__(self, data: Dict[str, Any]) -> List[Dict[str, Any]]:
         """
@@ -31,8 +35,30 @@ class EndpointHandler():
             A :obj:`list` | `dict`: will be serialized and returned
         """
 
-        # pseudo
         inputs = data.pop("inputs", data)
-        #self.model(input)
+        messages = [
+            {
+                "role": "user",
+                "content": ""
+                + inputs,
+            },
+        ]
+        encodeds = self.tokenizer.apply_chat_template(messages, return_tensors="pt")
+        encoded_length = len(encodeds[0])
+        model_inputs = encodeds.to('cuda')
+        result = self.model.generate(model_inputs,
+                                do_sample=False,
+                                output_scores=True,
+                                return_dict_in_generate=True,
+                                output_attentions=True,
+                                output_hidden_states=True,
+                                #num_beams=3,
+                                #no_repeat_ngram_size=1,
+                                early_stopping = True,
+                                #top_k=0,
+                                max_new_tokens=400)
+        x, logits_gen = result.sequences, result.scores
+        x = x[:,encoded_length:]
+        decoded = self.tokenizer.batch_decode(x)
 
-        return [{"outputs": inputs}]
+        return [{"outputs": decoded[0]}]