Hugging Face's logo

flexitok
/
mod-tokenizers-ltr_2digit

Safetensors
Undetermined
llama
tokenizer
bpe
flexitok
fineweb2
Model card Files
xet
Community
gsaltintas commited on
Commit
78f6415
·
verified ·
1 Parent(s): 3c5c2c5

Upload folder using huggingface_hub

Browse files
Files changed (2) hide show
  1. README.md +1 -1
  2. vocab.json +90 -90
README.md CHANGED
@@ -46,4 +46,4 @@ tokens = tokenizer.encode("Hello, world!")
46
  ## Sample Encoding
47
  | Text | Tokens | Token IDs |
48
  |------|--------|-----------|
49
- | `123450019 mod 67` | `12, 34, 50, 0, 1, 9, , mod, , 67` | `19, 41, 57, 7, 8, 16, 6, 4, 6, 74` |
 
46
  ## Sample Encoding
47
  | Text | Tokens | Token IDs |
48
  |------|--------|-----------|
49
+ | `1234500119 mod 67` | `12, 34, 50, 0, 1, 19, , mod, , 67` | `19, 41, 57, 7, 8, 26, 6, 4, 6, 74` |
vocab.json CHANGED
@@ -1,109 +1,109 @@
1
  {
 
 
 
 
 
 
 
 
 
 
2
  "11": 18,
3
- "62": 69,
4
- "89": 96,
5
- "75": 82,
6
- "</s>": 2,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
7
  "8": 15,
8
- "81": 88,
9
- "10": 17,
 
 
 
 
 
 
 
 
 
 
 
10
  "47": 54,
11
- "69": 76,
12
- "26": 33,
13
- "42": 49,
14
  "35": 42,
15
- "91": 98,
16
- "33": 40,
17
- "13": 20,
18
- "66": 73,
19
- "97": 104,
20
- "51": 58,
21
- "32": 39,
22
- "24": 31,
23
  "85": 92,
24
- "=": 5,
25
- "64": 71,
26
- "17": 24,
27
- "96": 103,
28
- "55": 62,
29
- "mod": 4,
30
- "25": 32,
31
- "56": 63,
32
  "88": 95,
33
- "5": 12,
34
- "45": 52,
35
- "70": 77,
 
 
 
36
  "77": 84,
37
- "59": 66,
38
- "2": 9,
39
- " ": 6,
40
- "31": 38,
41
- "78": 85,
42
- "60": 67,
43
- "37": 44,
44
- "19": 26,
45
- "38": 45,
46
- "50": 57,
47
  "28": 35,
48
- "68": 75,
49
- "46": 53,
50
- "94": 101,
51
- "99": 106,
 
 
52
  "95": 102,
53
- "53": 60,
54
- "52": 59,
55
- "23": 30,
56
- "<s>": 1,
57
- "9": 16,
58
- "18": 25,
59
- "4": 11,
60
- "74": 81,
61
- "1": 8,
62
- "20": 27,
63
  "48": 55,
 
 
 
64
  "36": 43,
65
- "22": 29,
66
- "72": 79,
67
- "30": 37,
68
- "12": 19,
69
- "82": 89,
70
- "40": 47,
71
- "83": 90,
72
- "73": 80,
73
  "49": 56,
74
- "84": 91,
75
- "16": 23,
76
- "29": 36,
77
- "21": 28,
78
- "0": 7,
79
  "43": 50,
80
- "39": 46,
81
- "80": 87,
82
  "27": 34,
83
- "71": 78,
84
- "7": 14,
85
- "79": 86,
86
- "41": 48,
87
- "34": 41,
88
- "67": 74,
89
  "90": 97,
90
- "<unk>": 0,
 
 
 
 
 
 
91
  "6": 13,
92
- "87": 94,
93
- "15": 22,
94
- "54": 61,
95
- "93": 100,
96
- "65": 72,
97
- "58": 65,
98
- "63": 70,
99
  "98": 105,
100
- "76": 83,
101
- "86": 93,
102
- "61": 68,
103
- "3": 10,
104
- "92": 99,
105
- "<pad>": 3,
106
- "57": 64,
107
- "14": 21,
108
- "44": 51
109
  }
 
1
  {
2
+ "66": 73,
3
+ "22": 29,
4
+ "<unk>": 0,
5
+ "79": 86,
6
+ "71": 78,
7
+ "18": 25,
8
+ "0": 7,
9
+ "16": 23,
10
+ "17": 24,
11
+ "45": 52,
12
  "11": 18,
13
+ "94": 101,
14
+ "32": 39,
15
+ "<s>": 1,
16
+ "30": 37,
17
+ "14": 21,
18
+ "80": 87,
19
+ "73": 80,
20
+ "57": 64,
21
+ "41": 48,
22
+ "37": 44,
23
+ "2": 9,
24
+ "20": 27,
25
+ "29": 36,
26
+ "26": 33,
27
+ "53": 60,
28
+ "39": 46,
29
+ "40": 47,
30
+ "38": 45,
31
+ "56": 63,
32
+ "64": 71,
33
+ "87": 94,
34
  "8": 15,
35
+ "23": 30,
36
+ "67": 74,
37
+ "51": 58,
38
+ "63": 70,
39
+ "89": 96,
40
+ "84": 91,
41
+ "=": 5,
42
+ "72": 79,
43
+ "60": 67,
44
+ "59": 66,
45
+ "21": 28,
46
+ "9": 16,
47
+ "82": 89,
48
  "47": 54,
49
+ "50": 57,
 
 
50
  "35": 42,
51
+ "15": 22,
52
+ "19": 26,
53
+ "44": 51,
 
 
 
 
 
54
  "85": 92,
55
+ "93": 100,
56
+ "31": 38,
 
 
 
 
 
 
57
  "88": 95,
58
+ "25": 32,
59
+ "62": 69,
60
+ "24": 31,
61
+ "99": 106,
62
+ "65": 72,
63
+ "74": 81,
64
  "77": 84,
65
+ "54": 61,
 
 
 
 
 
 
 
 
 
66
  "28": 35,
67
+ "3": 10,
68
+ "86": 93,
69
+ "83": 90,
70
+ " ": 6,
71
+ "42": 49,
72
+ "81": 88,
73
  "95": 102,
 
 
 
 
 
 
 
 
 
 
74
  "48": 55,
75
+ "68": 75,
76
+ "58": 65,
77
+ "91": 98,
78
  "36": 43,
79
+ "34": 41,
80
+ "52": 59,
81
+ "7": 14,
82
+ "78": 85,
83
+ "4": 11,
84
+ "13": 20,
85
+ "10": 17,
86
+ "61": 68,
87
  "49": 56,
88
+ "97": 104,
89
+ "75": 82,
90
+ "5": 12,
91
+ "12": 19,
92
+ "33": 40,
93
  "43": 50,
 
 
94
  "27": 34,
 
 
 
 
 
 
95
  "90": 97,
96
+ "mod": 4,
97
+ "1": 8,
98
+ "<pad>": 3,
99
+ "92": 99,
100
+ "76": 83,
101
+ "55": 62,
102
+ "96": 103,
103
  "6": 13,
104
+ "</s>": 2,
105
+ "70": 77,
106
+ "69": 76,
 
 
 
 
107
  "98": 105,
108
+ "46": 53
 
 
 
 
 
 
 
 
109
  }