handler.py

from transformers import AutoTokenizer

class EndpointHandler():
    def __init__(self, path=""):
        self.tokenizer =  AutoTokenizer.from_pretrained("otmanheddouch/shakespear-tokenizer")

    def __call__(self, data: str] -> Dict[str, Any]]:
        """
       data args:
            data:  text input
      Return:
            output (dict) : outputs with tokens and their ids
        """
        tokens = self.tokenizer.tokenize()
        tokens_ids = self.tokenizer.encode(example)
        
        return {"tokens:"tokens, "ids":tokens_ids}