run on A100

handler.py

@@ -25,7 +25,6 @@ class EndpointHandler:
             "/repository",
             device_map="auto",
             torch_dtype=dtype,
-            offload_folder="offload",
             local_files_only=True
         )
         gpu_info1 = nvmlDeviceGetMemoryInfo(gpu_h1)
@@ -52,7 +51,7 @@ class EndpointHandler:
                 top_k=10,
                 num_return_sequences=1,
                 eos_token_id=self.tokenizer.eos_token_id,
-                max_length=256,
+                max_length=256
             )
         # postprocess the prediction
         return prediction