Hugging Face's logo

jkgl
/
testing_v0

Text Generation
Transformers
Safetensors
llama
conversational
text-generation-inference
Model card Files
xet
Community
jkgl commited on
Commit
d606021
·
verified ·
1 Parent(s): 085b7f2

Upload custom tokenizer.json

Browse files
Files changed (1) hide show
  1. tokenizer.json +183 -0
tokenizer.json ADDED
@@ -0,0 +1,183 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "version": "1.0",
3
+ "model": {
4
+ "type": "BPE",
5
+ "vocab": {
6
+ "[PAD]": 0,
7
+ "<s>": 1,
8
+ "</s>": 2,
9
+ "[INST]": 3,
10
+ "[/INST]": 4,
11
+ "\t": 5,
12
+ "\n": 6,
13
+ " ": 7,
14
+ "!": 8,
15
+ "\"": 9,
16
+ "#": 10,
17
+ "$": 11,
18
+ "%": 12,
19
+ "&": 13,
20
+ "'": 14,
21
+ "(": 15,
22
+ ")": 16,
23
+ "*": 17,
24
+ "+": 18,
25
+ ",": 19,
26
+ "-": 20,
27
+ ".": 21,
28
+ "/": 22,
29
+ "0": 23,
30
+ "1": 24,
31
+ "2": 25,
32
+ "3": 26,
33
+ "4": 27,
34
+ "5": 28,
35
+ "6": 29,
36
+ "7": 30,
37
+ "8": 31,
38
+ "9": 32,
39
+ ":": 33,
40
+ ";": 34,
41
+ "<": 35,
42
+ "=": 36,
43
+ ">": 37,
44
+ "?": 38,
45
+ "@": 39,
46
+ "A": 40,
47
+ "B": 41,
48
+ "C": 42,
49
+ "D": 43,
50
+ "E": 44,
51
+ "F": 45,
52
+ "G": 46,
53
+ "H": 47,
54
+ "I": 48,
55
+ "J": 49,
56
+ "K": 50,
57
+ "L": 51,
58
+ "M": 52,
59
+ "N": 53,
60
+ "O": 54,
61
+ "P": 55,
62
+ "Q": 56,
63
+ "R": 57,
64
+ "S": 58,
65
+ "T": 59,
66
+ "U": 60,
67
+ "V": 61,
68
+ "W": 62,
69
+ "X": 63,
70
+ "Y": 64,
71
+ "Z": 65,
72
+ "[": 66,
73
+ "]": 67,
74
+ "^": 68,
75
+ "_": 69,
76
+ "`": 70,
77
+ "a": 71,
78
+ "b": 72,
79
+ "c": 73,
80
+ "d": 74,
81
+ "e": 75,
82
+ "f": 76,
83
+ "g": 77,
84
+ "h": 78,
85
+ "i": 79,
86
+ "j": 80,
87
+ "k": 81,
88
+ "l": 82,
89
+ "m": 83,
90
+ "n": 84,
91
+ "o": 85,
92
+ "p": 86,
93
+ "q": 87,
94
+ "r": 88,
95
+ "s": 89,
96
+ "t": 90,
97
+ "u": 91,
98
+ "v": 92,
99
+ "w": 93,
100
+ "x": 94,
101
+ "y": 95,
102
+ "z": 96,
103
+ "{": 97,
104
+ "|": 98,
105
+ "}": 99,
106
+ "~": 100
107
+ },
108
+ "merges": []
109
+ },
110
+ "pre_tokenizer": {
111
+ "type": "ByteLevel"
112
+ },
113
+ "decoder": {
114
+ "type": "ByteLevel"
115
+ },
116
+ "normalizer": {
117
+ "type": "Lowercase"
118
+ },
119
+ "post_processor": {
120
+ "type": "TemplateProcessing",
121
+ "single": [
122
+ {
123
+ "type": "SpecialToken",
124
+ "id": "<s>",
125
+ "ids": 1,
126
+ "tokens": [
127
+ "<s>"
128
+ ]
129
+ },
130
+ {
131
+ "type": "Sequence",
132
+ "text": " "
133
+ },
134
+ {
135
+ "type": "Feature",
136
+ "feature": "tokens"
137
+ },
138
+ {
139
+ "type": "SpecialToken",
140
+ "id": "</s>",
141
+ "ids": 2,
142
+ "tokens": [
143
+ "</s>"
144
+ ]
145
+ }
146
+ ],
147
+ "pair": [],
148
+ "special_tokens": {
149
+ "<s>": {
150
+ "id": "<s>",
151
+ "ids": 1,
152
+ "tokens": [
153
+ "<s>"
154
+ ]
155
+ },
156
+ "</s>": {
157
+ "id": "</s>",
158
+ "ids": 2,
159
+ "tokens": [
160
+ "</s>"
161
+ ]
162
+ }
163
+ }
164
+ },
165
+ "added_tokens": [
166
+ {
167
+ "id": 3,
168
+ "content": "[INST]",
169
+ "single_word": false,
170
+ "lstrip": false,
171
+ "rstrip": false,
172
+ "normalized": false
173
+ },
174
+ {
175
+ "id": 4,
176
+ "content": "[/INST]",
177
+ "single_word": false,
178
+ "lstrip": false,
179
+ "rstrip": false,
180
+ "normalized": false
181
+ }
182
+ ]
183
+ }