Add "add_eos_token" to tokenizer config; simplify usage

README.md

@@ -6892,10 +6892,7 @@ model = AutoModel.from_pretrained('intfloat/e5-mistral-7b-instruct')
 
 max_length = 4096
 # Tokenize the input texts
-batch_dict = tokenizer(input_texts, max_length=max_length - 1, return_attention_mask=False, padding=False, truncation=True)
-# append eos_token_id to every input_ids
-batch_dict['input_ids'] = [input_ids + [tokenizer.eos_token_id] for input_ids in batch_dict['input_ids']]
-batch_dict = tokenizer.pad(batch_dict, padding=True, return_attention_mask=True, return_tensors='pt')
+batch_dict = tokenizer(input_texts, max_length=max_length, padding=True, truncation=True, return_tensors='pt')
 
 outputs = model(**batch_dict)
 embeddings = last_token_pool(outputs.last_hidden_state, batch_dict['attention_mask'])

tokenizer_config.json

@@ -1,4 +1,5 @@
 {
+  "add_eos_token": true,
   "added_tokens_decoder": {
     "0": {
       "content": "<unk>",