Upload tokenizer
Browse files- tokenizer.json +16 -1
tokenizer.json
CHANGED
|
@@ -42,7 +42,22 @@
|
|
| 42 |
"<|pad|>": 1,
|
| 43 |
"+": 2,
|
| 44 |
"0": 3,
|
| 45 |
-
"1": 4
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 46 |
},
|
| 47 |
"merges": []
|
| 48 |
}
|
|
|
|
| 42 |
"<|pad|>": 1,
|
| 43 |
"+": 2,
|
| 44 |
"0": 3,
|
| 45 |
+
"1": 4,
|
| 46 |
+
"2": 5,
|
| 47 |
+
"3": 6,
|
| 48 |
+
"4": 7,
|
| 49 |
+
"5": 8,
|
| 50 |
+
"6": 9,
|
| 51 |
+
"7": 10,
|
| 52 |
+
"8": 11,
|
| 53 |
+
"9": 12,
|
| 54 |
+
":": 13,
|
| 55 |
+
"A": 14,
|
| 56 |
+
"B": 15,
|
| 57 |
+
"C": 16,
|
| 58 |
+
"D": 17,
|
| 59 |
+
"E": 18,
|
| 60 |
+
"F": 19
|
| 61 |
},
|
| 62 |
"merges": []
|
| 63 |
}
|