gsaltintas commited on
Commit
0f311c6
·
verified ·
1 Parent(s): 23c9c8f

Upload folder using huggingface_hub

Browse files
Files changed (2) hide show
  1. tokenizer_config.json +3 -2
  2. vocab.json +11 -11
tokenizer_config.json CHANGED
@@ -69,5 +69,6 @@
69
  "model_max_length": 1000000000000000019884624838656,
70
  "pad_token": "<pad>",
71
  "tokenizer_class": "PreTrainedTokenizerFast",
72
- "unk_token": "<unk>"
73
- }
 
 
69
  "model_max_length": 1000000000000000019884624838656,
70
  "pad_token": "<pad>",
71
  "tokenizer_class": "PreTrainedTokenizerFast",
72
+ "unk_token": "<unk>",
73
+ "number_handling": "individual"
74
+ }
vocab.json CHANGED
@@ -1,19 +1,19 @@
1
  {
2
- "<unk>": 0,
3
- "8": 15,
4
- "2": 9,
5
- "1": 8,
6
- "0": 7,
7
  "6": 13,
8
- "</s>": 2,
9
- "<s>": 1,
10
- "9": 16,
11
  "7": 14,
12
- "mod": 4,
13
- "<pad>": 3,
14
  " ": 6,
 
 
 
 
15
  "=": 5,
 
 
16
  "5": 12,
 
17
  "3": 10,
18
- "4": 11
 
19
  }
 
1
  {
 
 
 
 
 
2
  "6": 13,
3
+ "4": 11,
4
+ "0": 7,
 
5
  "7": 14,
 
 
6
  " ": 6,
7
+ "1": 8,
8
+ "<unk>": 0,
9
+ "9": 16,
10
+ "2": 9,
11
  "=": 5,
12
+ "<s>": 1,
13
+ "</s>": 2,
14
  "5": 12,
15
+ "mod": 4,
16
  "3": 10,
17
+ "8": 15,
18
+ "<pad>": 3
19
  }