Hugging Face's logo

almaghrabima
/
SARFTokenizer

Text Generation
Arabic
English
tokenizer
arabic
morphology
bpe
deeplatent
english
Model card Files
xet
 Community
almaghrabima commited on
Commit
a88f8a7
·
verified ·
1 Parent(s): 1e8911f

Upload benchmark_parallel_results.json with huggingface_hub

Browse files
Files changed (1) hide show
  1. benchmark_parallel_results.json +188 -0
benchmark_parallel_results.json ADDED
@@ -0,0 +1,188 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "version": "0.3.8",
3
+ "rust_available": true,
4
+ "cpu_count": 128,
5
+ "samples": 10000,
6
+ "total_bytes": 4353919,
7
+ "thread_counts": [
8
+ 1,
9
+ 2,
10
+ 4,
11
+ 8
12
+ ],
13
+ "runs": 3,
14
+ "results": {
15
+ "SARFTokenizer (HF)": {
16
+ "name": "SARFTokenizer (HF)",
17
+ "type": "sarf",
18
+ "vocab_size": 64641,
19
+ "benchmarks": [
20
+ {
21
+ "threads": 1,
22
+ "time_sec": 1.2484327759593725,
23
+ "texts_per_sec": 8010.04282534587,
24
+ "bytes_per_sec": 3487507.7648087065,
25
+ "mb_per_sec": 3.325946583565432
26
+ },
27
+ {
28
+ "threads": 2,
29
+ "time_sec": 0.24223483664294085,
30
+ "texts_per_sec": 41282.25377731365,
31
+ "bytes_per_sec": 17973958.908386767,
32
+ "mb_per_sec": 17.141302975069777,
33
+ "speedup": 5.1538118681070975
34
+ },
35
+ {
36
+ "threads": 4,
37
+ "time_sec": 0.25055884880324203,
38
+ "texts_per_sec": 39910.78362533811,
39
+ "bytes_per_sec": 17376831.91312485,
40
+ "mb_per_sec": 16.57183829605565,
41
+ "speedup": 4.982593039209473
42
+ },
43
+ {
44
+ "threads": 8,
45
+ "time_sec": 0.23175926196078458,
46
+ "texts_per_sec": 43148.22163047825,
47
+ "bytes_per_sec": 18786386.197315022,
48
+ "mb_per_sec": 17.916094014468214,
49
+ "speedup": 5.38676541078482
50
+ }
51
+ ],
52
+ "memory": {
53
+ "current_mb": 6.103515625e-05,
54
+ "peak_mb": 0.2074756622314453,
55
+ "texts_processed": 10000
56
+ }
57
+ },
58
+ "SARFTokenizer (Local)": {
59
+ "name": "SARFTokenizer (Local)",
60
+ "type": "sarf",
61
+ "vocab_size": 64641,
62
+ "benchmarks": [
63
+ {
64
+ "threads": 1,
65
+ "time_sec": 1.2962954323738813,
66
+ "texts_per_sec": 7714.2908555090635,
67
+ "bytes_per_sec": 3358739.7527327165,
68
+ "mb_per_sec": 3.2031438376738706
69
+ },
70
+ {
71
+ "threads": 2,
72
+ "time_sec": 0.24298642265299955,
73
+ "texts_per_sec": 41154.562838602105,
74
+ "bytes_per_sec": 17918363.307968363,
75
+ "mb_per_sec": 17.08828287884556,
76
+ "speedup": 5.3348471829023785
77
+ },
78
+ {
79
+ "threads": 4,
80
+ "time_sec": 0.23486537175873914,
81
+ "texts_per_sec": 42577.58359658189,
82
+ "bytes_per_sec": 18537935.019524623,
83
+ "mb_per_sec": 17.679152507328627,
84
+ "speedup": 5.51931271377662
85
+ },
86
+ {
87
+ "threads": 8,
88
+ "time_sec": 0.24369121653338274,
89
+ "texts_per_sec": 41035.53727645379,
90
+ "bytes_per_sec": 17866540.542316042,
91
+ "mb_per_sec": 17.038860838237802,
92
+ "speedup": 5.319417953647519
93
+ }
94
+ ],
95
+ "memory": {
96
+ "current_mb": 6.103515625e-05,
97
+ "peak_mb": 0.2074756622314453,
98
+ "texts_processed": 10000
99
+ }
100
+ },
101
+ "tiktoken (o200k)": {
102
+ "name": "tiktoken (o200k)",
103
+ "type": "tiktoken",
104
+ "vocab_size": 200019,
105
+ "benchmarks": [
106
+ {
107
+ "threads": 1,
108
+ "time_sec": 0.46731498713294667,
109
+ "texts_per_sec": 21398.842911826185,
110
+ "bytes_per_sec": 9316882.873181535,
111
+ "mb_per_sec": 8.885271905118499
112
+ },
113
+ {
114
+ "threads": 2,
115
+ "time_sec": 0.52791434712708,
116
+ "texts_per_sec": 18942.466811936807,
117
+ "bytes_per_sec": 8247396.61593611,
118
+ "mb_per_sec": 7.865330329834089,
119
+ "speedup": 0.8852098634486518
120
+ },
121
+ {
122
+ "threads": 4,
123
+ "time_sec": 0.48348489341636497,
124
+ "texts_per_sec": 20683.169497476425,
125
+ "bytes_per_sec": 9005284.465528306,
126
+ "mb_per_sec": 8.58810850670653,
127
+ "speedup": 0.966555508758175
128
+ },
129
+ {
130
+ "threads": 8,
131
+ "time_sec": 0.8265109478185574,
132
+ "texts_per_sec": 12099.053287065815,
133
+ "bytes_per_sec": 5267829.798856831,
134
+ "mb_per_sec": 5.023793982369262,
135
+ "speedup": 0.5654068931165998
136
+ }
137
+ ],
138
+ "memory": {
139
+ "current_mb": 0.0,
140
+ "peak_mb": 0.21294403076171875,
141
+ "texts_processed": 10000
142
+ }
143
+ },
144
+ "tiktoken (cl100k)": {
145
+ "name": "tiktoken (cl100k)",
146
+ "type": "tiktoken",
147
+ "vocab_size": 100277,
148
+ "benchmarks": [
149
+ {
150
+ "threads": 1,
151
+ "time_sec": 0.3374793001761039,
152
+ "texts_per_sec": 29631.447009584845,
153
+ "bytes_per_sec": 12901292.013252463,
154
+ "mb_per_sec": 12.303630841495956
155
+ },
156
+ {
157
+ "threads": 2,
158
+ "time_sec": 0.5219240722556909,
159
+ "texts_per_sec": 19159.875030828225,
160
+ "bytes_per_sec": 8342054.393434861,
161
+ "mb_per_sec": 7.955603021082745,
162
+ "speedup": 0.6466061216865516
163
+ },
164
+ {
165
+ "threads": 4,
166
+ "time_sec": 0.514385993902882,
167
+ "texts_per_sec": 19440.653747442506,
168
+ "bytes_per_sec": 8464303.172341112,
169
+ "mb_per_sec": 8.072188541737663,
170
+ "speedup": 0.6560818221652849
171
+ },
172
+ {
173
+ "threads": 8,
174
+ "time_sec": 0.8330062093834082,
175
+ "texts_per_sec": 12004.71243473924,
176
+ "bytes_per_sec": 5226754.555914744,
177
+ "mb_per_sec": 4.984621578135246,
178
+ "speedup": 0.4051341951291171
179
+ }
180
+ ],
181
+ "memory": {
182
+ "current_mb": 0.0,
183
+ "peak_mb": 0.4065704345703125,
184
+ "texts_processed": 10000
185
+ }
186
+ }
187
+ }
188
+ }