Hugging Face's logo

flexitok
/
mod-tokenizers-zero-padded-ltr_2digit

Safetensors
Undetermined
llama
tokenizer
bpe
flexitok
fineweb2
Model card Files
xet
 Community
gsaltintas commited on
Commit
7fef727
·
verified ·
1 Parent(s): 87eb7cb

Upload folder using huggingface_hub

Browse files
Files changed (2) hide show
  1. README.md +1 -1
  2. vocab.json +88 -88
README.md CHANGED
@@ -46,4 +46,4 @@ tokens = tokenizer.encode("Hello, world!")
46
  ## Sample Encoding
47
  | Text | Tokens | Token IDs |
48
  |------|--------|-----------|
49
- | `123500119 mod 67` | `12, 35, 0, 0, 11, 9, , mod, , 67` | `19, 42, 7, 7, 18, 16, 6, 4, 6, 74` |
 
46
  ## Sample Encoding
47
  | Text | Tokens | Token IDs |
48
  |------|--------|-----------|
49
+ | `103500109 mod 67` | `10, 35, 0, 0, 10, 9, , mod, , 67` | `17, 42, 7, 7, 17, 16, 6, 4, 6, 74` |
vocab.json CHANGED
@@ -1,109 +1,109 @@
1
  {
2
- "33": 40,
3
- "29": 36,
4
- "42": 49,
5
- "91": 98,
6
- "<unk>": 0,
7
- "10": 17,
8
- "89": 96,
9
- "17": 24,
10
- "43": 50,
11
- "97": 104,
12
- "50": 57,
13
- "74": 81,
14
- "60": 67,
15
- "65": 72,
16
- "mod": 4,
17
- "4": 11,
18
- "47": 54,
19
- "92": 99,
20
- "66": 73,
21
- "14": 21,
22
- " ": 6,
23
- "61": 68,
24
  "22": 29,
25
- "7": 14,
26
- "30": 37,
27
- "72": 79,
28
- "69": 76,
29
- "31": 38,
30
  "52": 59,
 
 
 
 
 
 
 
31
  "95": 102,
32
- "64": 71,
33
- "80": 87,
34
- "58": 65,
35
- "99": 106,
36
- "98": 105,
37
- "6": 13,
38
- "93": 100,
39
  "78": 85,
40
- "20": 27,
41
- "</s>": 2,
42
- "32": 39,
43
- "5": 12,
44
- "23": 30,
45
- "38": 45,
46
- "76": 83,
47
- "24": 31,
48
- "44": 51,
49
- "55": 62,
50
- "25": 32,
51
- "2": 9,
52
  "62": 69,
 
 
 
 
53
  "12": 19,
 
 
 
 
 
 
 
 
54
  "35": 42,
 
 
 
 
55
  "28": 35,
56
- "54": 61,
57
- "39": 46,
58
- "49": 56,
59
- "15": 22,
60
- "94": 101,
61
- "84": 91,
62
  "45": 52,
63
- "71": 78,
64
- "57": 64,
65
  "86": 93,
 
 
66
  "<s>": 1,
67
- "46": 53,
68
- "73": 80,
69
- "88": 95,
70
- "19": 26,
 
 
 
 
 
 
71
  "81": 88,
72
- "37": 44,
 
 
 
 
 
 
73
  "68": 75,
74
- "40": 47,
75
- "90": 97,
76
- "59": 66,
77
- "83": 90,
78
- "79": 86,
79
- "<pad>": 3,
80
  "13": 20,
81
- "96": 103,
82
- "70": 77,
83
- "53": 60,
84
- "82": 89,
85
- "34": 41,
86
- "3": 10,
87
- "41": 48,
88
- "21": 28,
89
- "48": 55,
90
- "85": 92,
91
- "51": 58,
92
- "16": 23,
93
- "9": 16,
94
- "63": 70,
95
  "1": 8,
 
 
96
  "26": 33,
97
- "75": 82,
98
- "67": 74,
 
 
 
99
  "87": 94,
100
- "77": 84,
101
- "11": 18,
102
- "18": 25,
103
  "8": 15,
104
- "56": 63,
 
 
 
 
 
 
 
 
 
 
 
 
105
  "27": 34,
106
- "36": 43,
107
- "0": 7,
108
- "=": 5
 
 
 
 
 
 
 
 
 
 
109
  }
 
1
  {
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2
  "22": 29,
3
+ "99": 106,
4
+ "48": 55,
 
 
 
5
  "52": 59,
6
+ "41": 48,
7
+ "15": 22,
8
+ "34": 41,
9
+ "50": 57,
10
+ "67": 74,
11
+ "5": 12,
12
+ "72": 79,
13
  "95": 102,
14
+ "85": 92,
15
+ "65": 72,
 
 
 
 
 
16
  "78": 85,
17
+ "73": 80,
 
 
 
 
 
 
 
 
 
 
 
18
  "62": 69,
19
+ "51": 58,
20
+ "74": 81,
21
+ "16": 23,
22
+ "39": 46,
23
  "12": 19,
24
+ "</s>": 2,
25
+ "47": 54,
26
+ "75": 82,
27
+ "46": 53,
28
+ "4": 11,
29
+ "64": 71,
30
+ "40": 47,
31
+ "36": 43,
32
  "35": 42,
33
+ "19": 26,
34
+ "9": 16,
35
+ "38": 45,
36
+ "69": 76,
37
  "28": 35,
 
 
 
 
 
 
38
  "45": 52,
39
+ "90": 97,
40
+ "94": 101,
41
  "86": 93,
42
+ "98": 105,
43
+ "21": 28,
44
  "<s>": 1,
45
+ "54": 61,
46
+ "77": 84,
47
+ "<pad>": 3,
48
+ "24": 31,
49
+ "7": 14,
50
+ "43": 50,
51
+ "44": 51,
52
+ "mod": 4,
53
+ "61": 68,
54
+ "60": 67,
55
  "81": 88,
56
+ "11": 18,
57
+ "17": 24,
58
+ "<unk>": 0,
59
+ "92": 99,
60
+ "31": 38,
61
+ "23": 30,
62
+ "96": 103,
63
  "68": 75,
64
+ "42": 49,
65
+ "93": 100,
66
+ "97": 104,
67
+ "32": 39,
68
+ "=": 5,
69
+ "20": 27,
70
  "13": 20,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
71
  "1": 8,
72
+ "10": 17,
73
+ "56": 63,
74
  "26": 33,
75
+ "3": 10,
76
+ "79": 86,
77
+ "0": 7,
78
+ "83": 90,
79
+ "82": 89,
80
  "87": 94,
 
 
 
81
  "8": 15,
82
+ "53": 60,
83
+ "29": 36,
84
+ "80": 87,
85
+ "91": 98,
86
+ "49": 56,
87
+ "30": 37,
88
+ "6": 13,
89
+ "59": 66,
90
+ "14": 21,
91
+ "66": 73,
92
+ "55": 62,
93
+ "25": 32,
94
+ "58": 65,
95
  "27": 34,
96
+ "76": 83,
97
+ "89": 96,
98
+ "63": 70,
99
+ " ": 6,
100
+ "57": 64,
101
+ "37": 44,
102
+ "33": 40,
103
+ "88": 95,
104
+ "18": 25,
105
+ "71": 78,
106
+ "84": 91,
107
+ "2": 9,
108
+ "70": 77
109
  }