Hugging Face's logo

Synthyra
/
Profluent-E1-150M

Fill-Mask
Transformers
Safetensors
E1
Model card Files
xet
Community
lhallee commited on
Commit
740a45c
·
verified ·
1 Parent(s): e0d0fbc

Upload tokenizer.json with huggingface_hub

Browse files
Files changed (1) hide show
  1. tokenizer.json +133 -0
tokenizer.json ADDED
@@ -0,0 +1,133 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "version": "1.0",
3
+ "truncation": null,
4
+ "padding": {
5
+ "strategy": "BatchLongest",
6
+ "direction": "Right",
7
+ "pad_to_multiple_of": null,
8
+ "pad_id": 0,
9
+ "pad_type_id": 0,
10
+ "pad_token": "<pad>"
11
+ },
12
+ "added_tokens": [
13
+ {
14
+ "id": 0,
15
+ "content": "<pad>",
16
+ "single_word": false,
17
+ "lstrip": false,
18
+ "rstrip": false,
19
+ "normalized": false,
20
+ "special": true
21
+ },
22
+ {
23
+ "id": 1,
24
+ "content": "<bos>",
25
+ "single_word": false,
26
+ "lstrip": false,
27
+ "rstrip": false,
28
+ "normalized": false,
29
+ "special": true
30
+ },
31
+ {
32
+ "id": 2,
33
+ "content": "<eos>",
34
+ "single_word": false,
35
+ "lstrip": false,
36
+ "rstrip": false,
37
+ "normalized": false,
38
+ "special": true
39
+ },
40
+ {
41
+ "id": 3,
42
+ "content": "<bos_glm>",
43
+ "single_word": false,
44
+ "lstrip": false,
45
+ "rstrip": false,
46
+ "normalized": false,
47
+ "special": true
48
+ },
49
+ {
50
+ "id": 4,
51
+ "content": "<eos_span>",
52
+ "single_word": false,
53
+ "lstrip": false,
54
+ "rstrip": false,
55
+ "normalized": false,
56
+ "special": true
57
+ },
58
+ {
59
+ "id": 5,
60
+ "content": "?",
61
+ "single_word": false,
62
+ "lstrip": false,
63
+ "rstrip": false,
64
+ "normalized": false,
65
+ "special": true
66
+ }
67
+ ],
68
+ "normalizer": null,
69
+ "pre_tokenizer": {
70
+ "type": "ByteLevel",
71
+ "add_prefix_space": false,
72
+ "trim_offsets": true,
73
+ "use_regex": true
74
+ },
75
+ "post_processor": {
76
+ "type": "ByteLevel",
77
+ "add_prefix_space": true,
78
+ "trim_offsets": true,
79
+ "use_regex": true
80
+ },
81
+ "decoder": {
82
+ "type": "ByteLevel",
83
+ "add_prefix_space": true,
84
+ "trim_offsets": true,
85
+ "use_regex": true
86
+ },
87
+ "model": {
88
+ "type": "BPE",
89
+ "dropout": null,
90
+ "unk_token": "X",
91
+ "continuing_subword_prefix": null,
92
+ "end_of_word_suffix": null,
93
+ "fuse_unk": false,
94
+ "byte_fallback": false,
95
+ "vocab": {
96
+ "<pad>": 0,
97
+ "<bos>": 1,
98
+ "<eos>": 2,
99
+ "<bos_glm>": 3,
100
+ "<eos_span>": 4,
101
+ "?": 5,
102
+ "1": 6,
103
+ "2": 7,
104
+ "A": 8,
105
+ "B": 9,
106
+ "C": 10,
107
+ "D": 11,
108
+ "E": 12,
109
+ "F": 13,
110
+ "G": 14,
111
+ "H": 15,
112
+ "I": 16,
113
+ "J": 17,
114
+ "K": 18,
115
+ "L": 19,
116
+ "M": 20,
117
+ "N": 21,
118
+ "O": 22,
119
+ "P": 23,
120
+ "Q": 24,
121
+ "R": 25,
122
+ "S": 26,
123
+ "T": 27,
124
+ "U": 28,
125
+ "V": 29,
126
+ "W": 30,
127
+ "X": 31,
128
+ "Y": 32,
129
+ "Z": 33
130
+ },
131
+ "merges": []
132
+ }
133
+ }