Spaces:
Running
Running
| <html> | |
| <head> | |
| <meta name="viewport" content="width=device-width, initial-scale=1.0"> | |
| <link rel="stylesheet" href="style.css" /> | |
| </head> | |
| <body> | |
| <h1>Some tokenizers</h1> | |
| <p>Powered by <a href="https://www.npmjs.com/package/@huggingface/tokenizers" target="_blank">Tokenizers.js</a>.</p> | |
| <h2>Input</h2> | |
| <textarea id="text" cols="40" rows="12"></textarea> | |
| <br> | |
| <select id="model"> | |
| <optgroup label="pika"> | |
| <option value="qikp/pika-5">pika 5</option> | |
| <option value="qikp/pika-4">pika 4</option> | |
| <option value="qikp/pika-3.1">pika 3.1</option> | |
| <option value="qikp/pika-3">pika 3</option> | |
| <option value="qikp/pika-2">pika 2</option> | |
| <option value="qikp/small-german-tokenizer">Small German Tokenizer</option> | |
| <option value="qikp/pika">pika</option> | |
| </optgroup> | |
| <optgroup label="EleutherAI"> | |
| <option value="EleutherAI/gpt-neox-20b">GPT-NeoX</option> | |
| </optgroup> | |
| <optgroup label="Silia"> | |
| <option value="qikp/strawberry-tokenizer">Strawberry (approximate)</option> | |
| </optgroup> | |
| <optgroup label="Supra Labs"> | |
| <option value="SupraLabs/MicroSupra-1k">MicroSupra</option> | |
| </optgroup> | |
| <optgroup label="OpenAI"> | |
| <option value="openai/gpt-oss-20b">gpt-oss</option> | |
| <option value="openai-community/openai-gpt">GPT-1</option> | |
| </optgroup> | |
| <optgroup label="Gemma"> | |
| <option value="unsloth/gemma-4-E2B">Gemma 4</option> | |
| </optgroup> | |
| <optgroup label="Qwen"> | |
| <option value="Qwen/Qwen3.5-0.8B">Qwen3.5</option> | |
| <option value="Qwen/Qwen3-0.6B">Qwen3</option> | |
| <option value="Qwen/Qwen2.5-0.5B">Qwen2.5</option> | |
| <option value="Qwen/Qwen1.5-0.5B">Qwen1.5</option> | |
| </optgroup> | |
| </select> | |
| <button id="tokenize">Tokenize</button> | |
| <br> | |
| <button id="clear-cache">Clear cache</button> | |
| <div id="status" hidden></div> | |
| <div id="progress-wrap" hidden> | |
| <progress id="progress" max="100" value="0"></progress> | |
| </div> | |
| <h2>Output</h2> | |
| <p id="totalLength"></p> | |
| <iframe id="output" src="about:blank" sandbox="allow-same-origin"></iframe> | |
| <script type="module" src="script.js"></script> | |
| </body> | |
| </html> |