Hugging Face's logo

WikiQuality
/
pre_filtered.yo

Fill-Mask
Transformers
Safetensors
deberta
Model card Files
xet
 Community
arturkulmizev commited on
Commit
39356e4
·
verified ·
1 Parent(s): 60ec033

Upload tokenizer

Browse files
Files changed (4) hide show
  1. special_tokens_map.json +5 -49
  2. tokenization.model +2 -2
  3. tokenizer.json +0 -0
  4. tokenizer_config.json +13 -33
special_tokens_map.json CHANGED
@@ -1,51 +1,7 @@
1
  {
2
- "bos_token": {
3
- "content": "[BOS]",
4
- "lstrip": false,
5
- "normalized": false,
6
- "rstrip": false,
7
- "single_word": false
8
- },
9
- "cls_token": {
10
- "content": "[CLS]",
11
- "lstrip": false,
12
- "normalized": false,
13
- "rstrip": false,
14
- "single_word": false
15
- },
16
- "eos_token": {
17
- "content": "[EOS]",
18
- "lstrip": false,
19
- "normalized": false,
20
- "rstrip": false,
21
- "single_word": false
22
- },
23
- "mask_token": {
24
- "content": "[MASK]",
25
- "lstrip": false,
26
- "normalized": false,
27
- "rstrip": false,
28
- "single_word": false
29
- },
30
- "pad_token": {
31
- "content": "[PAD]",
32
- "lstrip": false,
33
- "normalized": false,
34
- "rstrip": false,
35
- "single_word": false
36
- },
37
- "sep_token": {
38
- "content": "[SEP]",
39
- "lstrip": false,
40
- "normalized": false,
41
- "rstrip": false,
42
- "single_word": false
43
- },
44
- "unk_token": {
45
- "content": "[UNK]",
46
- "lstrip": false,
47
- "normalized": false,
48
- "rstrip": false,
49
- "single_word": false
50
- }
51
  }
 
1
  {
2
+ "bos_token": "<s>",
3
+ "eos_token": "</s>",
4
+ "mask_token": "<mask>",
5
+ "pad_token": "<pad>",
6
+ "unk_token": "<unk>"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
7
  }
tokenization.model CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:3da16fa4d665b548c31a35fb6565cd8d6cb426ad7d8bc24ca56603dd23342275
3
- size 310204
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e56fdd5b8446fb89495c442c45e76d551070dea7c946a7cc8a719db8c87c89a5
3
+ size 317947
tokenizer.json CHANGED
The diff for this file is too large to render. See raw diff
 
tokenizer_config.json CHANGED
@@ -4,7 +4,7 @@
4
  "add_prefix_space": true,
5
  "added_tokens_decoder": {
6
  "0": {
7
- "content": "[PAD]",
8
  "lstrip": false,
9
  "normalized": false,
10
  "rstrip": false,
@@ -12,7 +12,7 @@
12
  "special": true
13
  },
14
  "1": {
15
- "content": "[UNK]",
16
  "lstrip": false,
17
  "normalized": false,
18
  "rstrip": false,
@@ -20,7 +20,7 @@
20
  "special": true
21
  },
22
  "2": {
23
- "content": "[BOS]",
24
  "lstrip": false,
25
  "normalized": false,
26
  "rstrip": false,
@@ -28,7 +28,7 @@
28
  "special": true
29
  },
30
  "3": {
31
- "content": "[EOS]",
32
  "lstrip": false,
33
  "normalized": false,
34
  "rstrip": false,
@@ -36,48 +36,28 @@
36
  "special": true
37
  },
38
  "4": {
39
- "content": "[CLS]",
40
  "lstrip": false,
41
  "normalized": false,
42
  "rstrip": false,
43
  "single_word": false,
44
- "special": true
45
- },
46
- "5": {
47
- "content": "[SEP]",
48
- "lstrip": false,
49
- "normalized": false,
50
- "rstrip": false,
51
- "single_word": false,
52
- "special": true
53
- },
54
- "6": {
55
- "content": "[MASK]",
56
- "lstrip": false,
57
- "normalized": false,
58
- "rstrip": false,
59
- "single_word": false,
60
- "special": true
61
  }
62
  },
63
- "bos_token": "[BOS]",
64
  "bos_token_id": 2,
65
  "clean_up_tokenization_spaces": false,
66
- "cls_token": "[CLS]",
67
- "cls_token_id": 4,
68
- "eos_token": "[EOS]",
69
  "eos_token_id": 3,
70
  "legacy": true,
71
- "mask_token": "[MASK]",
72
- "mask_token_id": 6,
73
  "model_max_length": 1000000000000000019884624838656,
74
- "pad_token": "[PAD]",
75
  "pad_token_id": 0,
76
- "sep_token": "[SEP]",
77
- "sep_token_id": 5,
78
  "sp_model_kwargs": {},
79
  "spaces_between_special_tokens": false,
80
- "tokenizer_class": "HfTokenizerFromConfig",
81
- "unk_token": "[UNK]",
82
  "unk_token_id": 1
83
  }
 
4
  "add_prefix_space": true,
5
  "added_tokens_decoder": {
6
  "0": {
7
+ "content": "<pad>",
8
  "lstrip": false,
9
  "normalized": false,
10
  "rstrip": false,
 
12
  "special": true
13
  },
14
  "1": {
15
+ "content": "<unk>",
16
  "lstrip": false,
17
  "normalized": false,
18
  "rstrip": false,
 
20
  "special": true
21
  },
22
  "2": {
23
+ "content": "<s>",
24
  "lstrip": false,
25
  "normalized": false,
26
  "rstrip": false,
 
28
  "special": true
29
  },
30
  "3": {
31
+ "content": "</s>",
32
  "lstrip": false,
33
  "normalized": false,
34
  "rstrip": false,
 
36
  "special": true
37
  },
38
  "4": {
39
+ "content": "<mask>",
40
  "lstrip": false,
41
  "normalized": false,
42
  "rstrip": false,
43
  "single_word": false,
44
+ "special": false
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
45
  }
46
  },
47
+ "bos_token": "<s>",
48
  "bos_token_id": 2,
49
  "clean_up_tokenization_spaces": false,
50
+ "eos_token": "</s>",
 
 
51
  "eos_token_id": 3,
52
  "legacy": true,
53
+ "mask_token": "<mask>",
54
+ "mask_token_id": 4,
55
  "model_max_length": 1000000000000000019884624838656,
56
+ "pad_token": "<pad>",
57
  "pad_token_id": 0,
 
 
58
  "sp_model_kwargs": {},
59
  "spaces_between_special_tokens": false,
60
+ "tokenizer_class": "HfSentencePieceTokenizer",
61
+ "unk_token": "<unk>",
62
  "unk_token_id": 1
63
  }