Update app.py

app.py

@@ -9,7 +9,8 @@ TOKENIZER_PATH = os.environ.get("TOKENIZER_PATH", MODEL_PATH)
 
 tokenizer = AutoTokenizer.from_pretrained(TOKENIZER_PATH, trust_remote_code=True)
 
-model = AutoModel.from_pretrained(MODEL_PATH, trust_remote_code=True).float()
+# model = AutoModel.from_pretrained(MODEL_PATH, trust_remote_code=True).float()
+model = AutoModel.from_pretrained(MODEL_PATH, trust_remote_code=True, device_map="auto").eval()
 
 class StopOnTokens(StoppingCriteria):
     def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor, **kwargs) -> bool: