Updated handler.py

handler.py

@@ -1,37 +1,36 @@
 import torch
-import transformers
 from typing import Any, Dict
 from transformers import AutoModelForCausalLM, AutoTokenizer
 
 
-def load_pipeline(path):
-    tokenizer = AutoTokenizer.from_pretrained(path)
-    model = AutoModelForCausalLM.from_pretrained(path,
-                                                 torch_dtype=torch.bfloat16,
-                                                 trust_remote_code=True)
-    device = "cuda:0" if torch.cuda.is_available() else "cpu"
-    pipeline = transformers.pipeline('text-generation',
-                                     model=model,
-                                     tokenizer=tokenizer,
-                                     device=device)
-    return pipeline
-
-
 class EndpointHandler:
     def __init__(self, path=""):
-        self.path = path
-        self.pipeline = load_pipeline(self.path)
+        # load model and tokenizer from path
+        self.tokenizer = AutoTokenizer.from_pretrained(path)
+        self.model = AutoModelForCausalLM.from_pretrained(
+            path, device_map="auto", torch_dtype=torch.float16, trust_remote_code=True
+        )
+        self.device = "cuda" if torch.cuda.is_available() else "cpu"
 
     def __call__(self, data: Dict[str, Any]) -> Dict[str, str]:
+        # process input
         inputs = data.pop("inputs", data)
-        parameters = data.pop("parameters", {})
-        with torch.autocast(self.pipeline.device.type, dtype=torch.bfloat16):
-            try:
-                outputs = self.pipeline(inputs, **parameters, use_cache=True)
-                return outputs
-            except Exception as e:
-                print("Exception encounted. Reloading pipeline")
-                # Reload pipeline
-                self.pipeline = load_pipeline(self.path)
-                torch.cuda.empty_cache()
-                raise e
+        parameters = data.pop("parameters", None)
+
+        # preprocess
+        inputs = self.tokenizer(inputs, return_tensors="pt")
+        inputs.pop("token_type_ids")
+        inputs.pop("attention_mask")
+        inputs = inputs.to(self.device)
+
+        # pass inputs with all kwargs in data
+        if parameters is not None:
+            outputs = self.model.generate(**inputs, **parameters)
+        else:
+            outputs = self.model.generate(**inputs)
+
+        # postprocess the prediction
+        prediction = self.tokenizer.decode(
+            outputs[0], skip_special_tokens=True)
+
+        return [{"generated_text": prediction}]