Update index.html

index.html

@@ -146,37 +146,52 @@
         return;
       }
     
+    async function tokenize(){
+      const myRun = ++runId;
+    
+      if (!tokenizer) {
+        await loadTokenizer(modelSel.value);
+        if (!tokenizer) { render(); return; }
+      }
+    
+      // Make sure we always pass a string to encode()
+      const text = String(inputEl.value ?? '').trim();
+      if (!text) {
+        state.tokens = [];
+        state.ids = [];
+        render();
+        status('Type to tokenize…');
+        return;
+      }
+    
       status('Tokenizing…');
       try {
-        // 1) Always get IDs from encode
-        const enc = await tokenizer.encode(text, { add_special_tokens: false });
-        const ids =
-          (enc && (enc.ids ?? enc.input_ids ?? enc.inputIds)) || [];
+        // 1) Get IDs (no options arg!)
+        const enc = await tokenizer.encode(text);
+        // robustly pluck ids out of whatever shape the lib returns
+        let ids = (enc && (enc.ids ?? enc.input_ids ?? enc.inputIds)) || [];
     
-        // 2) Derive token strings in a version-tolerant way
-        let tokens = [];
+        // 2) Drop special tokens (e.g., BOS/EOS) for the demo
+        const specials = new Set(tokenizer.all_special_ids || []);
+        const idsNoSpecials = ids.filter(id => !specials.has(id));
     
-        // a) Preferred: convert_ids_to_tokens exists on many tokenizers
+        // 3) Turn IDs into token strings
+        let tokens = [];
         if (typeof tokenizer.convert_ids_to_tokens === 'function') {
-          tokens = tokenizer.convert_ids_to_tokens(ids);
-        }
-        // b) Fallback: id_to_token per-id
-        else if (typeof tokenizer.id_to_token === 'function') {
-          tokens = ids.map(id => tokenizer.id_to_token(id));
-        }
-        // c) Some builds include enc.tokens
-        else if (Array.isArray(enc.tokens)) {
-          tokens = enc.tokens;
-        }
-        // d) Last resort: stringify IDs (shouldn’t happen, but keeps UI stable)
-        else {
-          tokens = ids.map(String);
+          tokens = tokenizer.convert_ids_to_tokens(idsNoSpecials);
+        } else if (typeof tokenizer.id_to_token === 'function') {
+          tokens = idsNoSpecials.map(id => tokenizer.id_to_token(id));
+        } else if (Array.isArray(enc.tokens)) {
+          // fallback: some builds expose tokens directly (may include specials)
+          tokens = enc.tokens.filter((_, i) => !specials.has(ids[i]));
+        } else {
+          tokens = idsNoSpecials.map(String);
         }
     
-        if (myRun !== runId) return; // drop stale results
+        if (myRun !== runId) return; // drop stale result
     
         state.tokens = Array.isArray(tokens) ? tokens : [];
-        state.ids    = Array.isArray(ids)    ? ids    : [];
+        state.ids    = Array.isArray(idsNoSpecials) ? idsNoSpecials : [];
     
         render();
         status(`Done. ${state.tokens.length} tokens.`);