Hugging Face's logo

flexitok
/
mod-tokenizers-ltr_2digit

Safetensors
Undetermined
llama
tokenizer
bpe
flexitok
fineweb2
Model card Files
xet
Community
gsaltintas commited on
Commit
3c5c2c5
·
verified ·
1 Parent(s): 57756ab

Upload folder using huggingface_hub

Browse files
Files changed (2) hide show
  1. README.md +1 -1
  2. vocab.json +89 -89
README.md CHANGED
@@ -46,4 +46,4 @@ tokens = tokenizer.encode("Hello, world!")
46
  ## Sample Encoding
47
  | Text | Tokens | Token IDs |
48
  |------|--------|-----------|
49
- | `12345009 mod 67` | `12, 34, 50, 0, 9, , mod, , 67` | `19, 41, 57, 7, 16, 6, 4, 6, 74` |
 
46
  ## Sample Encoding
47
  | Text | Tokens | Token IDs |
48
  |------|--------|-----------|
49
+ | `123450019 mod 67` | `12, 34, 50, 0, 1, 9, , mod, , 67` | `19, 41, 57, 7, 8, 16, 6, 4, 6, 74` |
vocab.json CHANGED
@@ -1,109 +1,109 @@
1
  {
2
- "82": 89,
3
- "27": 34,
4
- "14": 21,
5
- "34": 41,
6
- "9": 16,
7
- "71": 78,
8
- "18": 25,
9
- "20": 27,
10
- "6": 13,
11
- "38": 45,
12
- "3": 10,
13
- "99": 106,
14
- "79": 86,
15
- "15": 22,
16
- "36": 43,
17
- "54": 61,
18
- "85": 92,
19
  "75": 82,
20
- "74": 81,
21
  "</s>": 2,
22
- "=": 5,
23
- "98": 105,
24
- "44": 51,
25
- "60": 67,
26
- "24": 31,
27
- "31": 38,
28
- "40": 47,
29
- "46": 53,
30
- "53": 60,
31
- "35": 42,
32
- "61": 68,
33
- "93": 100,
34
  "26": 33,
35
- "73": 80,
36
- "55": 62,
37
- "95": 102,
38
- "77": 84,
39
- "84": 91,
40
- "25": 32,
41
  "33": 40,
42
- "<s>": 1,
 
 
 
 
 
 
 
43
  "64": 71,
 
 
 
 
 
44
  "56": 63,
45
- "19": 26,
46
- "78": 85,
47
- "59": 66,
48
- "13": 20,
49
  "5": 12,
50
- "21": 28,
51
- "87": 94,
52
- "92": 99,
53
- "62": 69,
54
- "94": 101,
55
- "57": 64,
56
- "12": 19,
57
- "67": 74,
58
- "4": 11,
59
- "90": 97,
60
- "81": 88,
61
  "70": 77,
62
- "63": 70,
63
- "17": 24,
64
- "51": 58,
65
- "49": 56,
66
- "47": 54,
67
- "30": 37,
68
- "72": 79,
69
- "1": 8,
70
- "7": 14,
71
- "32": 39,
72
- "10": 17,
73
  " ": 6,
74
- "42": 49,
75
- "97": 104,
 
76
  "37": 44,
77
- "76": 83,
78
- "mod": 4,
79
  "50": 57,
80
- "91": 98,
81
  "68": 75,
82
- "23": 30,
83
- "58": 65,
84
- "<pad>": 3,
 
 
85
  "52": 59,
86
- "11": 18,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
87
  "83": 90,
 
 
 
 
88
  "29": 36,
89
- "28": 35,
90
- "88": 95,
91
- "22": 29,
92
- "48": 55,
93
- "89": 96,
94
- "69": 76,
95
- "2": 9,
96
- "65": 72,
97
- "8": 15,
98
- "96": 103,
99
- "41": 48,
100
  "43": 50,
101
  "39": 46,
102
- "86": 93,
103
- "45": 52,
104
  "80": 87,
105
- "0": 7,
106
- "66": 73,
107
- "16": 23,
108
- "<unk>": 0
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
109
  }
 
1
  {
2
+ "11": 18,
3
+ "62": 69,
4
+ "89": 96,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
5
  "75": 82,
 
6
  "</s>": 2,
7
+ "8": 15,
8
+ "81": 88,
9
+ "10": 17,
10
+ "47": 54,
11
+ "69": 76,
 
 
 
 
 
 
 
12
  "26": 33,
13
+ "42": 49,
14
+ "35": 42,
15
+ "91": 98,
 
 
 
16
  "33": 40,
17
+ "13": 20,
18
+ "66": 73,
19
+ "97": 104,
20
+ "51": 58,
21
+ "32": 39,
22
+ "24": 31,
23
+ "85": 92,
24
+ "=": 5,
25
  "64": 71,
26
+ "17": 24,
27
+ "96": 103,
28
+ "55": 62,
29
+ "mod": 4,
30
+ "25": 32,
31
  "56": 63,
32
+ "88": 95,
 
 
 
33
  "5": 12,
34
+ "45": 52,
 
 
 
 
 
 
 
 
 
 
35
  "70": 77,
36
+ "77": 84,
37
+ "59": 66,
38
+ "2": 9,
 
 
 
 
 
 
 
 
39
  " ": 6,
40
+ "31": 38,
41
+ "78": 85,
42
+ "60": 67,
43
  "37": 44,
44
+ "19": 26,
45
+ "38": 45,
46
  "50": 57,
47
+ "28": 35,
48
  "68": 75,
49
+ "46": 53,
50
+ "94": 101,
51
+ "99": 106,
52
+ "95": 102,
53
+ "53": 60,
54
  "52": 59,
55
+ "23": 30,
56
+ "<s>": 1,
57
+ "9": 16,
58
+ "18": 25,
59
+ "4": 11,
60
+ "74": 81,
61
+ "1": 8,
62
+ "20": 27,
63
+ "48": 55,
64
+ "36": 43,
65
+ "22": 29,
66
+ "72": 79,
67
+ "30": 37,
68
+ "12": 19,
69
+ "82": 89,
70
+ "40": 47,
71
  "83": 90,
72
+ "73": 80,
73
+ "49": 56,
74
+ "84": 91,
75
+ "16": 23,
76
  "29": 36,
77
+ "21": 28,
78
+ "0": 7,
 
 
 
 
 
 
 
 
 
79
  "43": 50,
80
  "39": 46,
 
 
81
  "80": 87,
82
+ "27": 34,
83
+ "71": 78,
84
+ "7": 14,
85
+ "79": 86,
86
+ "41": 48,
87
+ "34": 41,
88
+ "67": 74,
89
+ "90": 97,
90
+ "<unk>": 0,
91
+ "6": 13,
92
+ "87": 94,
93
+ "15": 22,
94
+ "54": 61,
95
+ "93": 100,
96
+ "65": 72,
97
+ "58": 65,
98
+ "63": 70,
99
+ "98": 105,
100
+ "76": 83,
101
+ "86": 93,
102
+ "61": 68,
103
+ "3": 10,
104
+ "92": 99,
105
+ "<pad>": 3,
106
+ "57": 64,
107
+ "14": 21,
108
+ "44": 51
109
  }