Hugging Face's logo

Spaces:
deepnevro
/
Tokenizer-Bench
Running

App Files Community
Tokenizer-Bench
File size: 4,014 Bytes
cc64f80
62a6af9
 
d296df9
 
 
 
 
 
62a6af9
 
 
d296df9
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3a58fd7
 
cc75b09
3a58fd7
70d8911
d296df9
 
 
 
70d8911
d296df9
 
 
 
 
 
 
 
 
 
 
 
 
 
 
62a6af9
d296df9
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Tokenizer Efficiency Leaderboard</title>
<link rel="stylesheet" href="style.css">
<link rel="preconnect" href="https://fonts.googleapis.com">
<link href="https://fonts.googleapis.com/css2?family=Kalam:wght@300;400;700&family=Inter:wght@400;500;600;700&display=swap" rel="stylesheet">
<base target="_blank">
</head>
<body>
<div class="paper-texture"></div>
<header class="site-header">
<div class="container">
<div class="header-content">
<div class="brand">
<div class="logo-sketch">
<svg viewBox="0 0 48 48" fill="none" stroke="currentColor" stroke-width="2.5" stroke-linecap="round">
<path d="M12 36 L24 12 L36 36" style="stroke-dasharray:4,3"/>
<circle cx="24" cy="24" r="18" style="stroke-dasharray:6,4"/>
</svg>
</div>
<div class="brand-text">
<h1>Tokenizer Bench</h1>
<p class="tagline">how many chars per token? let's find out.</p>
<p class="lede">How efficiently does each model turn text into tokens? Measured on identical text, in characters per token — higher is leaner: less cost, more context.</p>
</div>
</div>
</div>
</div>
</header>
<main class="container">
<!-- Stats Row -->
<section class="stats-grid" id="statsGrid">
<div class="stat-card sketch-box">
<div class="stat-value" id="statModels">—</div>
<div class="stat-label">models ranked</div>
</div>
<div class="stat-card sketch-box">
<div class="stat-value" id="statChars">—</div>
<div class="stat-label">total characters</div>
</div>
<div class="stat-card sketch-box highlight">
<div class="stat-value" id="statBest">—</div>
<div class="stat-label">most efficient</div>
</div>
</section>
<!-- Group Controls -->
<section class="controls-bar">
<div class="group-tabs" id="groupTabs"></div>
<div class="search-box sketch-input-wrap">
<input type="text" id="searchInput" placeholder="search models..." autocomplete="off">
</div>
</section>
<!-- Dataset Controls -->
<section class="controls-bar dataset-bar" id="datasetBar" style="display:none">
<div class="dataset-tabs" id="datasetTabs"></div>
</section>
<!-- Chart Section -->
<section class="chart-section" id="chartSection">
<div class="section-header sketch-underline">
<h2 id="chartTitle">Efficiency Sketch</h2>
<span class="section-badge" id="chartBadge">chars / token</span>
</div>
<div class="chart-container sketch-box" id="chartContainer">
<div class="chart-placeholder">loading data...</div>
</div>
</section>
<!-- Table Section -->
<section class="table-section">
<div class="section-header sketch-underline">
<h2>Leaderboard</h2>
<div class="table-meta">
<span class="timestamp" id="timestamp">—</span>
</div>
</div>
<div class="table-wrap sketch-box">
<table class="leaderboard" id="leaderboard">
<thead>
<tr>
<th class="col-rank" data-sort="rank">#</th>
<th class="col-model" data-sort="model">Model</th>
<th class="col-backend" data-sort="backend">Backend</th>
<th class="col-vocab" data-sort="vocab">Vocab</th>
<th class="col-tokens" data-sort="tokens">Tokens</th>
<th class="col-cpt" data-sort="cpt">C/T</th>
<th class="col-price" data-sort="price" title="Per Million Output Token Price">$/M Token</th>
<th class="col-price-adj" data-sort="price_adj" title="Per Million Characters, Price Adjusted (÷ C/T)">$/M Chars</th>
<th class="col-throughput" data-sort="throughput" title="Throughput">Tokens/sec</th>
<th class="col-throughput-adj" data-sort="throughput_adj" title="Effective Throughput in Characters/sec (× C/T)">Chars/sec</th>
<th class="col-tax" data-sort="tax" title="Extra tokens vs the most efficient tokenizer in this view">Tokenizer Inefficiency</th>
</tr>
</thead>
<tbody id="tableBody">
<tr class="loading-row">
<td colspan="11">
<div class="sketch-spinner"></div>
<p>loading benchmark data...</p>
</td>
</tr>
</tbody>
</table>
</div>
</section>
</main>
<footer class="site-footer">
<div class="container">
<p>tokenizer benchmark suite</p>
</div>
</footer>
<script src="script.js"></script>
</body>
</html>