gsaltintas commited on
Commit
6634830
·
verified ·
1 Parent(s): 0f311c6

Upload folder using huggingface_hub

Browse files
Files changed (2) hide show
  1. README.md +1 -1
  2. vocab.json +13 -13
README.md CHANGED
@@ -46,4 +46,4 @@ tokens = tokenizer.encode("Hello, world!")
46
  ## Sample Encoding
47
  | Text | Tokens | Token IDs |
48
  |------|--------|-----------|
49
- | `12345009 mod 67` | `1, 2, 3, 4, 5, 0, 0, 9, , mod, , 6, 7` | `8, 9, 10, 11, 12, 7, 7, 16, 6, 4, 6, 13, 14` |
 
46
  ## Sample Encoding
47
  | Text | Tokens | Token IDs |
48
  |------|--------|-----------|
49
+ | `123450019 mod 67` | `1, 2, 3, 4, 5, 0, 0, 1, 9, , mod, , 6, 7` | `8, 9, 10, 11, 12, 7, 7, 8, 16, 6, 4, 6, 13, 14` |
vocab.json CHANGED
@@ -1,19 +1,19 @@
1
  {
2
- "6": 13,
3
- "4": 11,
4
- "0": 7,
5
- "7": 14,
6
- " ": 6,
7
- "1": 8,
8
- "<unk>": 0,
9
- "9": 16,
10
- "2": 9,
11
  "=": 5,
12
- "<s>": 1,
13
  "</s>": 2,
 
14
  "5": 12,
 
 
 
 
 
 
15
  "mod": 4,
16
- "3": 10,
17
- "8": 15,
18
- "<pad>": 3
 
 
19
  }
 
1
  {
2
+ "3": 10,
 
 
 
 
 
 
 
 
3
  "=": 5,
 
4
  "</s>": 2,
5
+ "8": 15,
6
  "5": 12,
7
+ "7": 14,
8
+ " ": 6,
9
+ "<s>": 1,
10
+ "<pad>": 3,
11
+ "4": 11,
12
+ "0": 7,
13
  "mod": 4,
14
+ "9": 16,
15
+ "<unk>": 0,
16
+ "2": 9,
17
+ "6": 13,
18
+ "1": 8
19
  }