Upload folder using huggingface_hub
Browse files- README.md +1 -1
- vocab.json +11 -11
README.md
CHANGED
|
@@ -46,4 +46,4 @@ tokens = tokenizer.encode("Hello, world!")
|
|
| 46 |
## Sample Encoding
|
| 47 |
| Text | Tokens | Token IDs |
|
| 48 |
|------|--------|-----------|
|
| 49 |
-
| `
|
|
|
|
| 46 |
## Sample Encoding
|
| 47 |
| Text | Tokens | Token IDs |
|
| 48 |
|------|--------|-----------|
|
| 49 |
+
| `103500109 mod 67` | `1, 0, 3, 5, 0, 0, 1, 0, 9, , mod, , 6, 7` | `8, 7, 10, 12, 7, 7, 8, 7, 16, 6, 4, 6, 13, 14` |
|
vocab.json
CHANGED
|
@@ -1,19 +1,19 @@
|
|
| 1 |
{
|
| 2 |
-
"
|
| 3 |
-
"
|
| 4 |
-
"0": 7,
|
| 5 |
-
"5": 12,
|
| 6 |
"7": 14,
|
| 7 |
-
"
|
|
|
|
|
|
|
|
|
|
| 8 |
"<pad>": 3,
|
|
|
|
|
|
|
| 9 |
"6": 13,
|
| 10 |
-
"
|
| 11 |
" ": 6,
|
| 12 |
"<unk>": 0,
|
| 13 |
-
"
|
| 14 |
"=": 5,
|
| 15 |
-
"
|
| 16 |
-
"</s>": 2,
|
| 17 |
-
"mod": 4,
|
| 18 |
-
"8": 15
|
| 19 |
}
|
|
|
|
| 1 |
{
|
| 2 |
+
"2": 9,
|
| 3 |
+
"</s>": 2,
|
|
|
|
|
|
|
| 4 |
"7": 14,
|
| 5 |
+
"8": 15,
|
| 6 |
+
"5": 12,
|
| 7 |
+
"9": 16,
|
| 8 |
+
"4": 11,
|
| 9 |
"<pad>": 3,
|
| 10 |
+
"0": 7,
|
| 11 |
+
"mod": 4,
|
| 12 |
"6": 13,
|
| 13 |
+
"1": 8,
|
| 14 |
" ": 6,
|
| 15 |
"<unk>": 0,
|
| 16 |
+
"<s>": 1,
|
| 17 |
"=": 5,
|
| 18 |
+
"3": 10
|
|
|
|
|
|
|
|
|
|
| 19 |
}
|