Spaces:
Running
Running
| <html lang="en"> | |
| <head> | |
| <meta charset="UTF-8"> | |
| <meta name="viewport" content="width=device-width, initial-scale=1.0"> | |
| <title>Tokenizer Efficiency Leaderboard</title> | |
| <link rel="stylesheet" href="style.css"> | |
| <link rel="preconnect" href="https://fonts.googleapis.com"> | |
| <link href="https://fonts.googleapis.com/css2?family=Kalam:wght@300;400;700&family=Inter:wght@400;500;600;700&display=swap" rel="stylesheet"> | |
| <base target="_blank"> | |
| </head> | |
| <body> | |
| <div class="paper-texture"></div> | |
| <header class="site-header"> | |
| <div class="container"> | |
| <div class="header-content"> | |
| <div class="brand"> | |
| <div class="logo-sketch"> | |
| <svg viewBox="0 0 48 48" fill="none" stroke="currentColor" stroke-width="2.5" stroke-linecap="round"> | |
| <path d="M12 36 L24 12 L36 36" style="stroke-dasharray:4,3"/> | |
| <circle cx="24" cy="24" r="18" style="stroke-dasharray:6,4"/> | |
| </svg> | |
| </div> | |
| <div class="brand-text"> | |
| <h1>Tokenizer Bench</h1> | |
| <p class="tagline">how many chars per token? let's find out.</p> | |
| <p class="lede">How efficiently does each model turn text into tokens? Measured on identical text, in characters per token — higher is leaner: less cost, more context.</p> | |
| </div> | |
| </div> | |
| </div> | |
| </div> | |
| </header> | |
| <main class="container"> | |
| <!-- Stats Row --> | |
| <section class="stats-grid" id="statsGrid"> | |
| <div class="stat-card sketch-box"> | |
| <div class="stat-value" id="statModels">—</div> | |
| <div class="stat-label">models ranked</div> | |
| </div> | |
| <div class="stat-card sketch-box"> | |
| <div class="stat-value" id="statChars">—</div> | |
| <div class="stat-label">total characters</div> | |
| </div> | |
| <div class="stat-card sketch-box highlight"> | |
| <div class="stat-value" id="statBest">—</div> | |
| <div class="stat-label">most efficient</div> | |
| </div> | |
| </section> | |
| <!-- Group Controls --> | |
| <section class="controls-bar"> | |
| <div class="group-tabs" id="groupTabs"></div> | |
| <div class="search-box sketch-input-wrap"> | |
| <input type="text" id="searchInput" placeholder="search models..." autocomplete="off"> | |
| </div> | |
| </section> | |
| <!-- Dataset Controls --> | |
| <section class="controls-bar dataset-bar" id="datasetBar" style="display:none"> | |
| <div class="dataset-tabs" id="datasetTabs"></div> | |
| </section> | |
| <!-- Chart Section --> | |
| <section class="chart-section" id="chartSection"> | |
| <div class="section-header sketch-underline"> | |
| <h2 id="chartTitle">Efficiency Sketch</h2> | |
| <span class="section-badge" id="chartBadge">chars / token</span> | |
| </div> | |
| <div class="chart-container sketch-box" id="chartContainer"> | |
| <div class="chart-placeholder">loading data...</div> | |
| </div> | |
| </section> | |
| <!-- Table Section --> | |
| <section class="table-section"> | |
| <div class="section-header sketch-underline"> | |
| <h2>Leaderboard</h2> | |
| <div class="table-meta"> | |
| <span class="timestamp" id="timestamp">—</span> | |
| </div> | |
| </div> | |
| <div class="table-wrap sketch-box"> | |
| <table class="leaderboard" id="leaderboard"> | |
| <thead> | |
| <tr> | |
| <th class="col-rank" data-sort="rank">#</th> | |
| <th class="col-model" data-sort="model">Model</th> | |
| <th class="col-backend" data-sort="backend">Backend</th> | |
| <th class="col-vocab" data-sort="vocab">Vocab</th> | |
| <th class="col-tokens" data-sort="tokens">Tokens</th> | |
| <th class="col-cpt" data-sort="cpt">C/T</th> | |
| <th class="col-price" data-sort="price" title="Per Million Output Token Price">$/M Token</th> | |
| <th class="col-price-adj" data-sort="price_adj" title="Per Million Characters, Price Adjusted (÷ C/T)">$/M Chars</th> | |
| <th class="col-throughput" data-sort="throughput" title="Throughput">Tokens/sec</th> | |
| <th class="col-throughput-adj" data-sort="throughput_adj" title="Effective Throughput in Characters/sec (× C/T)">Chars/sec</th> | |
| <th class="col-tax" data-sort="tax" title="Extra tokens vs the most efficient tokenizer in this view">Tokenizer Inefficiency</th> | |
| </tr> | |
| </thead> | |
| <tbody id="tableBody"> | |
| <tr class="loading-row"> | |
| <td colspan="11"> | |
| <div class="sketch-spinner"></div> | |
| <p>loading benchmark data...</p> | |
| </td> | |
| </tr> | |
| </tbody> | |
| </table> | |
| </div> | |
| </section> | |
| </main> | |
| <footer class="site-footer"> | |
| <div class="container"> | |
| <p>tokenizer benchmark suite</p> | |
| </div> | |
| </footer> | |
| <script src="script.js"></script> | |
| </body> | |
| </html> |