Hugging Face's logo

lhallee
/
csm8

Fill-Mask
Transformers
Safetensors
esm
Model card Files
xet
 Community
lhallee commited on
Commit
49af715
·
verified ·
1 Parent(s): d076d89

Upload tokenizer

Browse files
Files changed (3) hide show
  1. special_tokens_map.json +37 -0
  2. tokenizer_config.json +52 -0
  3. vocab.txt +95 -0
special_tokens_map.json ADDED
@@ -0,0 +1,37 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "cls_token": {
3
+ "content": "<cls>",
4
+ "lstrip": false,
5
+ "normalized": false,
6
+ "rstrip": false,
7
+ "single_word": false
8
+ },
9
+ "eos_token": {
10
+ "content": "<eos>",
11
+ "lstrip": false,
12
+ "normalized": false,
13
+ "rstrip": false,
14
+ "single_word": false
15
+ },
16
+ "mask_token": {
17
+ "content": "<mask>",
18
+ "lstrip": false,
19
+ "normalized": false,
20
+ "rstrip": false,
21
+ "single_word": false
22
+ },
23
+ "pad_token": {
24
+ "content": "<pad>",
25
+ "lstrip": false,
26
+ "normalized": false,
27
+ "rstrip": false,
28
+ "single_word": false
29
+ },
30
+ "unk_token": {
31
+ "content": "<unk>",
32
+ "lstrip": false,
33
+ "normalized": false,
34
+ "rstrip": false,
35
+ "single_word": false
36
+ }
37
+ }
tokenizer_config.json ADDED
@@ -0,0 +1,52 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "added_tokens_decoder": {
3
+ "0": {
4
+ "content": "<cls>",
5
+ "lstrip": false,
6
+ "normalized": false,
7
+ "rstrip": false,
8
+ "single_word": false,
9
+ "special": true
10
+ },
11
+ "1": {
12
+ "content": "<pad>",
13
+ "lstrip": false,
14
+ "normalized": false,
15
+ "rstrip": false,
16
+ "single_word": false,
17
+ "special": true
18
+ },
19
+ "2": {
20
+ "content": "<eos>",
21
+ "lstrip": false,
22
+ "normalized": false,
23
+ "rstrip": false,
24
+ "single_word": false,
25
+ "special": true
26
+ },
27
+ "3": {
28
+ "content": "<unk>",
29
+ "lstrip": false,
30
+ "normalized": false,
31
+ "rstrip": false,
32
+ "single_word": false,
33
+ "special": true
34
+ },
35
+ "4": {
36
+ "content": "<mask>",
37
+ "lstrip": false,
38
+ "normalized": false,
39
+ "rstrip": false,
40
+ "single_word": false,
41
+ "special": true
42
+ }
43
+ },
44
+ "clean_up_tokenization_spaces": true,
45
+ "cls_token": "<cls>",
46
+ "eos_token": "<eos>",
47
+ "mask_token": "<mask>",
48
+ "model_max_length": 1000000000000000019884624838656,
49
+ "pad_token": "<pad>",
50
+ "tokenizer_class": "EsmTokenizer",
51
+ "unk_token": "<unk>"
52
+ }
vocab.txt ADDED
@@ -0,0 +1,95 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ <cls>
2
+ <pad>
3
+ <eos>
4
+ <unk>
5
+ <mask>
6
+ [L]
7
+ [A]
8
+ [G]
9
+ [V]
10
+ [S]
11
+ [E]
12
+ [R]
13
+ [T]
14
+ [I]
15
+ [D]
16
+ [P]
17
+ [K]
18
+ [Q]
19
+ [N]
20
+ [F]
21
+ [Y]
22
+ [M]
23
+ [H]
24
+ [W]
25
+ [C]
26
+ [X]
27
+ [B]
28
+ [U]
29
+ [Z]
30
+ [O]
31
+ [*]
32
+ a
33
+ A
34
+ @
35
+ b
36
+ B
37
+ #
38
+ $
39
+ %
40
+ r
41
+ R
42
+ n
43
+ N
44
+ d
45
+ D
46
+ c
47
+ C
48
+ e
49
+ E
50
+ q
51
+ Q
52
+ ^
53
+ G
54
+ &
55
+ g
56
+ h
57
+ H
58
+ i
59
+ I
60
+ j
61
+ +
62
+ M
63
+ m
64
+ l
65
+ J
66
+ L
67
+ k
68
+ K
69
+ (
70
+ f
71
+ F
72
+ p
73
+ P
74
+ o
75
+ O
76
+ =
77
+ s
78
+ z
79
+ Z
80
+ w
81
+ S
82
+ X
83
+ T
84
+ t
85
+ x
86
+ W
87
+ y
88
+ Y
89
+ u
90
+ v
91
+ U
92
+ V
93
+ ]
94
+ }
95
+ )