handler.py

from typing import Dict, List, Any
from transformers import (
    AutoModelForCausalLM,
    AutoTokenizer)
import torch

model = AutoModelForCausalLM.from_pretrained(
                "sjster/test_medium",
                trust_remote_code=True,
                quantization_config=None,
                torch_dtype=torch.float,     # data type is float
                device_map="auto",
            )

class EndpointHandler():
    def __init__(self, path=""):
        # Preload all the elements you are going to need at inference.
         self.model = AutoModelForCausalLM.from_pretrained(
                path,
                trust_remote_code=True,
                quantization_config=None,
                torch_dtype=torch.float,     # data type is float
                device_map="auto",

    def __call__(self, data: Dict[str, Any]) -> List[Dict[str, Any]]:
        """
       data args:
            inputs (:obj: `str` | `PIL.Image` | `np.array`)
            kwargs
      Return:
            A :obj:`list` | `dict`: will be serialized and returned
        """

        # pseudo
        inputs = data.pop("inputs", data)
        #self.model(input)

        return [{"outputs": inputs}]