Hugging Face's logo

flexitok
/
mod-tokenizers-rtl_2digit

Safetensors
Undetermined
llama
tokenizer
bpe
flexitok
fineweb2
Model card Files
xet
Community
gsaltintas commited on
Commit
9219f18
·
verified ·
1 Parent(s): 119cad1

Upload folder using huggingface_hub

Browse files
Files changed (2) hide show
  1. README.md +1 -1
  2. vocab.json +90 -90
README.md CHANGED
@@ -46,4 +46,4 @@ tokens = tokenizer.encode("Hello, world!")
46
  ## Sample Encoding
47
  | Text | Tokens | Token IDs |
48
  |------|--------|-----------|
49
- | `12345009 mod 67` | `12, 34, 50, 0, 9, , mod, , 67` | `20, 42, 58, 7, 16, 6, 4, 6, 75` |
 
46
  ## Sample Encoding
47
  | Text | Tokens | Token IDs |
48
  |------|--------|-----------|
49
+ | `123450019 mod 67` | `12, 34, 50, 0, 19, , mod, , 67` | `20, 42, 58, 7, 27, 6, 4, 6, 75` |
vocab.json CHANGED
@@ -1,109 +1,109 @@
1
  {
2
- "32": 40,
3
- "12": 20,
4
- "13": 21,
5
- "38": 46,
6
- "78": 86,
7
- "60": 68,
8
- "82": 90,
9
- "85": 93,
10
- "70": 78,
11
- "95": 103,
12
- "21": 29,
13
- "48": 56,
14
- "<s>": 1,
15
  "4": 11,
16
- "56": 64,
17
- "Ġ": 17,
18
- "36": 44,
 
 
 
19
  "20": 28,
20
- "24": 32,
21
- "42": 50,
22
- "17": 25,
23
  "45": 53,
 
24
  "49": 57,
25
- "98": 106,
26
- "3": 10,
27
- "22": 30,
28
- "79": 87,
29
- "1": 8,
30
- "44": 52,
31
- "52": 60,
32
- "71": 79,
33
- "7": 14,
34
- "65": 73,
35
  "89": 97,
36
- "31": 39,
37
- "37": 45,
38
- "33": 41,
39
- "75": 83,
40
  "66": 74,
41
- "87": 95,
42
- "74": 82,
 
 
 
 
 
 
 
 
 
43
  "92": 100,
44
- "mod": 4,
45
- "50": 58,
46
- "10": 18,
47
- "77": 85,
48
- "23": 31,
49
- "58": 66,
50
- "14": 22,
51
- "43": 51,
52
- "93": 101,
53
- "2": 9,
54
  "72": 80,
55
- "61": 69,
56
- "91": 99,
57
- "<unk>": 0,
58
- "46": 54,
59
- "97": 105,
60
- "9": 16,
61
- "64": 72,
62
- "30": 38,
63
- "29": 37,
64
- "53": 61,
65
- "68": 76,
66
- "55": 63,
67
  "90": 98,
68
- "57": 65,
69
  "39": 47,
 
 
 
 
 
 
70
  "6": 13,
71
- "83": 91,
72
- "59": 67,
73
- "69": 77,
74
  "80": 88,
75
- "35": 43,
76
- "76": 84,
77
- "11": 19,
78
- "34": 42,
79
- "40": 48,
80
- "</s>": 2,
81
- "0": 7,
82
- "8": 15,
83
- "96": 104,
84
- "54": 62,
85
- "84": 92,
86
- "41": 49,
87
- "15": 23,
88
- "5": 12,
89
- "81": 89,
90
- "19": 27,
91
  " ": 6,
92
- "28": 36,
93
- "<pad>": 3,
 
 
 
 
 
 
 
 
 
94
  "27": 35,
95
- "47": 55,
96
- "16": 24,
97
  "73": 81,
98
- "86": 94,
 
 
 
 
 
 
 
 
 
 
99
  "63": 71,
100
- "18": 26,
101
- "26": 34,
102
- "88": 96,
103
- "67": 75,
104
- "94": 102,
105
- "25": 33,
 
 
 
 
 
 
106
  "62": 70,
107
- "51": 59,
108
- "=": 5
 
 
 
 
 
 
 
 
109
  }
 
1
  {
2
+ "40": 48,
3
+ "37": 45,
4
+ "53": 61,
5
+ "5": 12,
6
+ "29": 37,
7
+ "91": 99,
8
+ "68": 76,
9
+ "55": 63,
10
+ "75": 83,
11
+ "97": 105,
 
 
 
12
  "4": 11,
13
+ "47": 55,
14
+ "3": 10,
15
+ "<s>": 1,
16
+ "87": 95,
17
+ "88": 96,
18
+ "7": 14,
19
  "20": 28,
20
+ "Ġ": 17,
 
 
21
  "45": 53,
22
+ "57": 65,
23
  "49": 57,
24
+ "76": 84,
25
+ "19": 27,
26
+ "69": 77,
27
+ "23": 31,
28
+ "41": 49,
29
+ "8": 15,
 
 
 
 
30
  "89": 97,
31
+ "17": 25,
 
 
 
32
  "66": 74,
33
+ "65": 73,
34
+ "18": 26,
35
+ "84": 92,
36
+ "51": 59,
37
+ "67": 75,
38
+ "21": 29,
39
+ "56": 64,
40
+ "38": 46,
41
+ "44": 52,
42
+ "28": 36,
43
+ "26": 34,
44
  "92": 100,
 
 
 
 
 
 
 
 
 
 
45
  "72": 80,
46
+ "31": 39,
 
 
 
 
 
 
 
 
 
 
 
47
  "90": 98,
48
+ "52": 60,
49
  "39": 47,
50
+ "50": 58,
51
+ "81": 89,
52
+ "94": 102,
53
+ "13": 21,
54
+ "25": 33,
55
+ "9": 16,
56
  "6": 13,
 
 
 
57
  "80": 88,
58
+ "98": 106,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
59
  " ": 6,
60
+ "48": 56,
61
+ "33": 41,
62
+ "46": 54,
63
+ "71": 79,
64
+ "12": 20,
65
+ "96": 104,
66
+ "10": 18,
67
+ "59": 67,
68
+ "34": 42,
69
+ "74": 82,
70
+ "58": 66,
71
  "27": 35,
72
+ "70": 78,
 
73
  "73": 81,
74
+ "54": 62,
75
+ "=": 5,
76
+ "83": 91,
77
+ "mod": 4,
78
+ "82": 90,
79
+ "60": 68,
80
+ "2": 9,
81
+ "95": 103,
82
+ "93": 101,
83
+ "77": 85,
84
+ "79": 87,
85
  "63": 71,
86
+ "85": 93,
87
+ "0": 7,
88
+ "30": 38,
89
+ "35": 43,
90
+ "42": 50,
91
+ "22": 30,
92
+ "<unk>": 0,
93
+ "64": 72,
94
+ "1": 8,
95
+ "<pad>": 3,
96
+ "61": 69,
97
+ "16": 24,
98
  "62": 70,
99
+ "15": 23,
100
+ "32": 40,
101
+ "86": 94,
102
+ "24": 32,
103
+ "11": 19,
104
+ "78": 86,
105
+ "</s>": 2,
106
+ "43": 51,
107
+ "14": 22,
108
+ "36": 44
109
  }