a-albiol commited on
Commit
1a10fb8
·
verified ·
1 Parent(s): ee0a52b

Upload tokenizer

Browse files
Files changed (2) hide show
  1. tokenizer.json +2 -4
  2. tokenizer_config.json +9 -0
tokenizer.json CHANGED
@@ -2,14 +2,12 @@
2
  "version": "1.0",
3
  "truncation": {
4
  "direction": "Right",
5
- "max_length": 128,
6
  "strategy": "LongestFirst",
7
  "stride": 0
8
  },
9
  "padding": {
10
- "strategy": {
11
- "Fixed": 128
12
- },
13
  "direction": "Right",
14
  "pad_to_multiple_of": null,
15
  "pad_id": 0,
 
2
  "version": "1.0",
3
  "truncation": {
4
  "direction": "Right",
5
+ "max_length": 512,
6
  "strategy": "LongestFirst",
7
  "stride": 0
8
  },
9
  "padding": {
10
+ "strategy": "BatchLongest",
 
 
11
  "direction": "Right",
12
  "pad_to_multiple_of": null,
13
  "pad_id": 0,
tokenizer_config.json CHANGED
@@ -1,4 +1,5 @@
1
  {
 
2
  "added_tokens_decoder": {
3
  "0": {
4
  "content": "<pad>",
@@ -930,9 +931,17 @@
930
  "clean_up_tokenization_spaces": false,
931
  "eos_token": "</s>",
932
  "extra_ids": 100,
 
 
933
  "model_max_length": 512,
 
934
  "pad_token": "<pad>",
 
 
935
  "sp_model_kwargs": {},
 
936
  "tokenizer_class": "T5Tokenizer",
 
 
937
  "unk_token": "<unk>"
938
  }
 
1
  {
2
+ "add_prefix_space": null,
3
  "added_tokens_decoder": {
4
  "0": {
5
  "content": "<pad>",
 
931
  "clean_up_tokenization_spaces": false,
932
  "eos_token": "</s>",
933
  "extra_ids": 100,
934
+ "extra_special_tokens": {},
935
+ "max_length": 128,
936
  "model_max_length": 512,
937
+ "pad_to_multiple_of": null,
938
  "pad_token": "<pad>",
939
+ "pad_token_type_id": 0,
940
+ "padding_side": "right",
941
  "sp_model_kwargs": {},
942
+ "stride": 0,
943
  "tokenizer_class": "T5Tokenizer",
944
+ "truncation_side": "right",
945
+ "truncation_strategy": "longest_first",
946
  "unk_token": "<unk>"
947
  }