gsaltintas commited on
Commit
ec8e436
·
verified ·
1 Parent(s): a8c283f

Upload folder using huggingface_hub

Browse files
Files changed (2) hide show
  1. README.md +1 -1
  2. vocab.json +11 -11
README.md CHANGED
@@ -46,4 +46,4 @@ tokens = tokenizer.encode("Hello, world!")
46
  ## Sample Encoding
47
  | Text | Tokens | Token IDs |
48
  |------|--------|-----------|
49
- | `123500119 mod 67` | `1, 2, 3, 5, 0, 0, 1, 1, 9, , mod, , 6, 7` | `8, 9, 10, 12, 7, 7, 8, 8, 16, 6, 4, 6, 13, 14` |
 
46
  ## Sample Encoding
47
  | Text | Tokens | Token IDs |
48
  |------|--------|-----------|
49
+ | `103500109 mod 67` | `1, 0, 3, 5, 0, 0, 1, 0, 9, , mod, , 6, 7` | `8, 7, 10, 12, 7, 7, 8, 7, 16, 6, 4, 6, 13, 14` |
vocab.json CHANGED
@@ -1,19 +1,19 @@
1
  {
2
- "<s>": 1,
3
- "9": 16,
4
- "0": 7,
5
- "5": 12,
6
  "7": 14,
7
- "1": 8,
 
 
 
8
  "<pad>": 3,
 
 
9
  "6": 13,
10
- "2": 9,
11
  " ": 6,
12
  "<unk>": 0,
13
- "3": 10,
14
  "=": 5,
15
- "4": 11,
16
- "</s>": 2,
17
- "mod": 4,
18
- "8": 15
19
  }
 
1
  {
2
+ "2": 9,
3
+ "</s>": 2,
 
 
4
  "7": 14,
5
+ "8": 15,
6
+ "5": 12,
7
+ "9": 16,
8
+ "4": 11,
9
  "<pad>": 3,
10
+ "0": 7,
11
+ "mod": 4,
12
  "6": 13,
13
+ "1": 8,
14
  " ": 6,
15
  "<unk>": 0,
16
+ "<s>": 1,
17
  "=": 5,
18
+ "3": 10
 
 
 
19
  }