Hugging Face's logo

Xerv-AI
/
CRAYON-tokenizer

Text Generation
English
generic
tokenizers
hardware-accelerated
avx2
cuda
rocm
double-array-trie
crayon
Model card Files
xet
 Community
CRAYON-tokenizer
File size: 3,733 Bytes
708f4a3
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
{
  "date": "2026-02-02T21:46:22.756992",
  "test_text_bytes": 30800,
  "iterations": 10,
  "results": [
    {
      "name": "CRAYON (CPU - code)",
      "status": "OK",
      "vocab_size": "~250k",
      "avg_tokens": 30800.0,
      "token_count": 30800,
      "load_time_ms": 128.97940003313124,
      "avg_time_ms": 1.2961800443008542,
      "min_time_ms": 1.007900107651949,
      "max_time_ms": 2.303199842572212,
      "tokens_per_sec": 23762130.990539353,
      "mb_per_sec": 22.661334028758386
    },
    {
      "name": "CRAYON (CPU - science)",
      "status": "OK",
      "vocab_size": "~250k",
      "avg_tokens": 24900.0,
      "token_count": 24900,
      "load_time_ms": 3.807599889114499,
      "avg_time_ms": 1.3703399803489447,
      "min_time_ms": 0.9711999446153641,
      "max_time_ms": 2.43859994225204,
      "tokens_per_sec": 18170673.232243754,
      "mb_per_sec": 21.43494998798246
    },
    {
      "name": "CRAYON (CPU - lite)",
      "status": "OK",
      "vocab_size": "50k",
      "avg_tokens": 15700.0,
      "token_count": 15700,
      "load_time_ms": 20.62970004044473,
      "avg_time_ms": 1.5809000004082918,
      "min_time_ms": 1.2891001533716917,
      "max_time_ms": 1.9415998831391335,
      "tokens_per_sec": 9931051.929878697,
      "mb_per_sec": 18.580029690509473
    },
    {
      "name": "tiktoken (p50k/GPT-3)",
      "status": "OK",
      "vocab_size": 50000,
      "avg_tokens": 11900.0,
      "token_count": 11900,
      "load_time_ms": 0.008899951353669167,
      "avg_time_ms": 28.1568999402225,
      "min_time_ms": 21.030299831181765,
      "max_time_ms": 55.71989994496107,
      "tokens_per_sec": 422631.7536825385,
      "mb_per_sec": 1.0431961262664624
    },
    {
      "name": "tiktoken (cl100k/GPT-4)",
      "status": "OK",
      "vocab_size": 100000,
      "avg_tokens": 9000.0,
      "token_count": 9000,
      "load_time_ms": 0.011600088328123093,
      "avg_time_ms": 23.468929948285222,
      "min_time_ms": 20.06639982573688,
      "max_time_ms": 35.85169999860227,
      "tokens_per_sec": 383485.74135386146,
      "mb_per_sec": 1.2515768298783763
    },
    {
      "name": "HF T5 (SentencePiece)",
      "status": "OK",
      "vocab_size": 32000,
      "avg_tokens": 12601.0,
      "token_count": 12601,
      "load_time_ms": 1777.7703001629561,
      "avg_time_ms": 32.928459998220205,
      "min_time_ms": 32.26630017161369,
      "max_time_ms": 34.046499989926815,
      "tokens_per_sec": 382678.08457125194,
      "mb_per_sec": 0.8920298412649765
    },
    {
      "name": "HF LLaMA (SP-BPE)",
      "status": "OK",
      "vocab_size": 32000,
      "avg_tokens": 11401.0,
      "token_count": 11401,
      "load_time_ms": 1174.7749999631196,
      "avg_time_ms": 39.65424003545195,
      "min_time_ms": 30.960500007495284,
      "max_time_ms": 45.88270001113415,
      "tokens_per_sec": 287510.2382445661,
      "mb_per_sec": 0.7407321113467842
    },
    {
      "name": "HF GPT-2 (BPE)",
      "status": "OK",
      "vocab_size": 50257,
      "avg_tokens": 15700.0,
      "token_count": 15700,
      "load_time_ms": 1819.5615001022816,
      "avg_time_ms": 73.55678000021726,
      "min_time_ms": 61.30379997193813,
      "max_time_ms": 98.4288000036031,
      "tokens_per_sec": 213440.55571700702,
      "mb_per_sec": 0.3993264651501295
    },
    {
      "name": "HF BERT (WordPiece)",
      "status": "OK",
      "vocab_size": 30522,
      "avg_tokens": 11402.0,
      "token_count": 11402,
      "load_time_ms": 1832.9594999086112,
      "avg_time_ms": 58.81147999316454,
      "min_time_ms": 50.545900128781796,
      "max_time_ms": 68.34379979409277,
      "tokens_per_sec": 193873.71311392295,
      "mb_per_sec": 0.49944617868359115
    }
  ]
}