Hugging Face's logo

flexitok
/
mod-tokenizers-ltr_2digit

Safetensors
Undetermined
llama
tokenizer
bpe
flexitok
fineweb2
Model card Files
xet
Community
gsaltintas commited on
Commit
6c6f734
·
verified ·
1 Parent(s): 78f6415

Upload folder using huggingface_hub

Browse files
Files changed (3) hide show
  1. README.md +1 -1
  2. tokenizer.json +1 -1
  3. vocab.json +91 -91
README.md CHANGED
@@ -46,4 +46,4 @@ tokens = tokenizer.encode("Hello, world!")
46
  ## Sample Encoding
47
  | Text | Tokens | Token IDs |
48
  |------|--------|-----------|
49
- | `1234500119 mod 67` | `12, 34, 50, 0, 1, 19, , mod, , 67` | `19, 41, 57, 7, 8, 26, 6, 4, 6, 74` |
 
46
  ## Sample Encoding
47
  | Text | Tokens | Token IDs |
48
  |------|--------|-----------|
49
+ | `123500119 mod 67` | `12, 35, 0, 0, 11, 9, , mod, , 67` | `19, 42, 7, 7, 18, 16, 6, 4, 6, 74` |
tokenizer.json CHANGED
@@ -83,7 +83,7 @@
83
  "type": "ByteLevel",
84
  "add_prefix_space": false,
85
  "trim_offsets": true,
86
- "use_regex": true
87
  }
88
  ]
89
  },
 
83
  "type": "ByteLevel",
84
  "add_prefix_space": false,
85
  "trim_offsets": true,
86
+ "use_regex": false
87
  }
88
  ]
89
  },
vocab.json CHANGED
@@ -1,109 +1,109 @@
1
  {
2
- "66": 73,
3
- "22": 29,
4
- "<unk>": 0,
5
- "79": 86,
6
- "71": 78,
7
- "18": 25,
8
  "0": 7,
9
- "16": 23,
10
- "17": 24,
11
- "45": 52,
12
- "11": 18,
13
- "94": 101,
 
14
  "32": 39,
15
- "<s>": 1,
16
- "30": 37,
17
- "14": 21,
18
- "80": 87,
19
- "73": 80,
20
- "57": 64,
21
  "41": 48,
22
- "37": 44,
23
- "2": 9,
24
- "20": 27,
25
- "29": 36,
26
- "26": 33,
27
- "53": 60,
28
- "39": 46,
29
- "40": 47,
30
- "38": 45,
31
- "56": 63,
32
- "64": 71,
33
- "87": 94,
34
- "8": 15,
35
- "23": 30,
36
- "67": 74,
37
- "51": 58,
38
- "63": 70,
39
- "89": 96,
40
- "84": 91,
41
- "=": 5,
42
- "72": 79,
43
- "60": 67,
44
- "59": 66,
45
- "21": 28,
46
- "9": 16,
47
- "82": 89,
48
- "47": 54,
49
  "50": 57,
50
- "35": 42,
51
- "15": 22,
52
- "19": 26,
53
- "44": 51,
54
  "85": 92,
55
- "93": 100,
56
- "31": 38,
57
- "88": 95,
58
  "25": 32,
59
- "62": 69,
60
- "24": 31,
61
  "99": 106,
62
- "65": 72,
 
 
63
  "74": 81,
64
- "77": 84,
65
- "54": 61,
66
- "28": 35,
67
- "3": 10,
68
- "86": 93,
69
  "83": 90,
70
- " ": 6,
71
- "42": 49,
72
- "81": 88,
73
- "95": 102,
74
- "48": 55,
75
- "68": 75,
76
- "58": 65,
77
- "91": 98,
78
- "36": 43,
79
- "34": 41,
80
- "52": 59,
81
- "7": 14,
82
- "78": 85,
83
- "4": 11,
84
- "13": 20,
85
- "10": 17,
86
- "61": 68,
87
- "49": 56,
88
- "97": 104,
89
  "75": 82,
90
- "5": 12,
91
- "12": 19,
 
 
 
 
 
 
 
 
 
 
 
 
92
  "33": 40,
 
 
93
  "43": 50,
94
- "27": 34,
95
- "90": 97,
96
- "mod": 4,
97
- "1": 8,
98
- "<pad>": 3,
99
- "92": 99,
100
  "76": 83,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
101
  "55": 62,
102
- "96": 103,
103
  "6": 13,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
104
  "</s>": 2,
105
- "70": 77,
106
- "69": 76,
107
- "98": 105,
108
- "46": 53
 
 
 
 
 
 
109
  }
 
1
  {
2
+ "56": 63,
3
+ "93": 100,
4
+ "30": 37,
5
+ "51": 58,
 
 
6
  "0": 7,
7
+ "79": 86,
8
+ "96": 103,
9
+ "37": 44,
10
+ "92": 99,
11
+ "95": 102,
12
+ "7": 14,
13
  "32": 39,
 
 
 
 
 
 
14
  "41": 48,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
15
  "50": 57,
16
+ "71": 78,
 
 
 
17
  "85": 92,
18
+ "59": 66,
 
 
19
  "25": 32,
20
+ "11": 18,
 
21
  "99": 106,
22
+ "26": 33,
23
+ "88": 95,
24
+ "1": 8,
25
  "74": 81,
26
+ "84": 91,
27
+ "46": 53,
 
 
 
28
  "83": 90,
29
+ "64": 71,
30
+ "21": 28,
31
+ "=": 5,
32
+ "69": 76,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
33
  "75": 82,
34
+ "44": 51,
35
+ "mod": 4,
36
+ "40": 47,
37
+ "68": 75,
38
+ "42": 49,
39
+ "28": 35,
40
+ "94": 101,
41
+ "90": 97,
42
+ "65": 72,
43
+ "70": 77,
44
+ "66": 73,
45
+ "22": 29,
46
+ "98": 105,
47
+ "86": 93,
48
  "33": 40,
49
+ "<unk>": 0,
50
+ "24": 31,
51
  "43": 50,
 
 
 
 
 
 
52
  "76": 83,
53
+ "38": 45,
54
+ "17": 24,
55
+ "82": 89,
56
+ "47": 54,
57
+ "61": 68,
58
+ "18": 25,
59
+ "73": 80,
60
+ "34": 41,
61
+ "67": 74,
62
+ "97": 104,
63
+ "<s>": 1,
64
+ "49": 56,
65
+ "39": 46,
66
+ "78": 85,
67
+ "29": 36,
68
+ "89": 96,
69
  "55": 62,
 
70
  "6": 13,
71
+ "52": 59,
72
+ "45": 52,
73
+ "3": 10,
74
+ "4": 11,
75
+ "54": 61,
76
+ "<pad>": 3,
77
+ "53": 60,
78
+ "19": 26,
79
+ "10": 17,
80
+ "58": 65,
81
+ "27": 34,
82
+ "91": 98,
83
+ "14": 21,
84
+ "23": 30,
85
+ "48": 55,
86
+ "62": 69,
87
+ " ": 6,
88
+ "35": 42,
89
+ "31": 38,
90
+ "2": 9,
91
+ "77": 84,
92
+ "72": 79,
93
+ "36": 43,
94
+ "60": 67,
95
+ "80": 87,
96
+ "16": 23,
97
+ "9": 16,
98
  "</s>": 2,
99
+ "63": 70,
100
+ "8": 15,
101
+ "5": 12,
102
+ "81": 88,
103
+ "13": 20,
104
+ "57": 64,
105
+ "87": 94,
106
+ "20": 27,
107
+ "15": 22,
108
+ "12": 19
109
  }