Add handler.py for Inference Endpoints

handler.py

@@ -15,6 +15,7 @@ class EndpointHandler:
         inputs = data.pop("inputs", data)
         parameters = data.pop("parameters", {})
         
+        # Handle chat format
         if isinstance(inputs, list) and len(inputs) > 0 and isinstance(inputs[0], dict):
             text = self.tokenizer.apply_chat_template(
                 inputs, 
@@ -26,11 +27,13 @@ class EndpointHandler:
         
         encoded = self.tokenizer(text, return_tensors="pt").to(self.model.device)
         
+        # Default generation parameters
         gen_kwargs = {
             "max_new_tokens": parameters.get("max_new_tokens", 512),
             "temperature": parameters.get("temperature", 0.7),
             "top_p": parameters.get("top_p", 0.9),
             "do_sample": parameters.get("do_sample", True),
+            "pad_token_id": self.tokenizer.eos_token_id,
         }
         
         with torch.no_grad():
@@ -38,9 +41,10 @@ class EndpointHandler:
         
         decoded = self.tokenizer.decode(outputs[0], skip_special_tokens=True)
         
-        if "<|im_start|>assistant" in decoded:
-            decoded = decoded.split("<|im_start|>assistant")[-1].strip()
-            if decoded.endswith("<|im_end|>"):
-                decoded = decoded[:-10].strip()
+        # Return only the generated part (remove input)
+        if isinstance(inputs, str):
+            generated = decoded[len(inputs):].strip()
+        else:
+            generated = decoded
         
-        return [{"generated_text": decoded}]
+        return [{"generated_text": generated}]