Update README.md
Browse files
README.md
CHANGED
|
@@ -78,7 +78,7 @@ input_texts = queries + documents
|
|
| 78 |
max_length = 512
|
| 79 |
|
| 80 |
# Tokenize the input texts
|
| 81 |
-
batch_dict = tokenizer(input_texts, max_length=max_length, padding=True, truncation=True, return_tensors='pt')
|
| 82 |
|
| 83 |
# Important! Adding EOS token at the end
|
| 84 |
batch_dict['input_ids'] = [input_ids + [tokenizer.eos_token_id] for input_ids in batch_dict['input_ids']]
|
|
|
|
| 78 |
max_length = 512
|
| 79 |
|
| 80 |
# Tokenize the input texts
|
| 81 |
+
batch_dict = tokenizer(input_texts, max_length=max_length-1, padding=True, truncation=True, return_tensors='pt')
|
| 82 |
|
| 83 |
# Important! Adding EOS token at the end
|
| 84 |
batch_dict['input_ids'] = [input_ids + [tokenizer.eos_token_id] for input_ids in batch_dict['input_ids']]
|