Update tokenizer.py

tokenizer.py

@@ -102,7 +102,6 @@ class Tokenizer(BaseTokenizer):
             eos_idx = ids.index(self.eos_id)
         except ValueError:
             eos_idx = len(ids)
-        # Truncate after EOS
         ids = ids[:eos_idx]
-        probs = probs[:eos_idx + 1]  # but include prob. for EOS (if it exists)
+        probs = probs[:eos_idx + 1]  
         return probs, ids