inference_mode versus no_grad

app.py

@@ -69,7 +69,6 @@ class Translators:
         # model = AutoModel.from_pretrained(self.model_name, trust_remote_code=True)
         # model.half() # recommended for GPU
         model.eval()
-        # model.float()
         # Translating from one or several sentences to a sole language
         src_tokens = tokenizer.encode_source_tokens_to_input_ids(self.input_text, target_language=self.tl)
         # src_tokens may be a torch.Tensor or dict depending on tokenizer; ensure it's a tensor
@@ -79,14 +78,13 @@ class Translators:
         #     # if tokenizer returns dict-like inputs (input_ids, attention_mask)
         #     for k, v in src_tokens.items():
         #         src_tokens[k] = v.to(self.device)
-        # src_tokens = src_tokens.to(self.device)
         # generated_tokens = model.generate(src_tokens)
         # return tokenizer.batch_decode(generated_tokens, skip_special_tokens=True)[0] 
         # Translating from one or several sentences to corresponding languages
         # src_tokens = tokenizer.encode_source_tokens_to_input_ids_with_different_tags([english_text, english_text, ], target_languages_list=["de", "zh", ])      
         # generated_tokens = model.generate(src_tokens.to(self.device))
         # results = tokenizer.batch_decode(generated_tokens, skip_special_tokens=True)
-        with torch.no_grad(): # no_grad inference_mode
+        with torch.inference_mode(): # no_grad inference_mode
             generated_tokens = model.generate(src_tokens)
         result = tokenizer.batch_decode(generated_tokens, skip_special_tokens=True)[0]
         return result