Update handler.py

handler.py

@@ -25,7 +25,14 @@ class EndpointHandler:
         model_name = model_dir2 #"./json_extraction_all"  # Pretrained model for sentiment analysis
         self.tokenizer = T5Tokenizer.from_pretrained(model_name)
         
-        self.model = T5ForConditionalGeneration.from_pretrained(model_name, device_map="auto")
+        #self.model = T5ForConditionalGeneration.from_pretrained(model_name, device_map="auto")
+        # Load model in float16 for faster inference
+        self.model = T5ForConditionalGeneration.from_pretrained(
+            model_name,
+            torch_dtype=torch.float16,  # Use float16 for faster computation
+            device_map="auto"  # Automatically uses GPU if available
+        )
+        
         self.model.eval()  # Set model to evaluation mode (no training)
 
         # Check if the model is on GPU