Update index.html

index.html

@@ -30,6 +30,10 @@
     .chip{border:1px solid var(--chip-border);background:var(--chip);padding:.35rem .5rem;border-radius:10px;font-family:var(--mono);font-size:.9rem;transition:background .12s,border-color .12s}
     .chip:hover{background:var(--chip-hover);border-color:var(--accent)}
     .chip.active{outline:2px solid var(--accent)}
+    .chip.special {
+      border-color: #38bdf8;
+      background: #0b2235;
+    }
     pre.ids{font-family:var(--mono);background:#0a1220;border:1px solid var(--border);border-radius:10px;padding:.75rem;max-height:360px;overflow:auto;white-space:pre-wrap}
     .caption{color:var(--muted);font-size:.9rem;margin-top:.5rem}
     footer{color:var(--muted);text-align:center;padding:1.25rem 0 2rem}
@@ -148,36 +152,49 @@
     
       status('Tokenizing…');
       try {
-        const enc = await tokenizer.encode(text); // returns EITHER an array OR an object, depending on tokenizer
-      
-        // >>> handle both shapes
-        let ids = Array.isArray(enc)
+        const enc = await tokenizer.encode(text); // include specials (default)
+        // Handle both array/object return shapes
+        const ids = Array.isArray(enc)
           ? enc
           : (enc && (enc.ids ?? enc.input_ids ?? enc.inputIds)) || [];
       
-        // Drop special tokens (for GPT-2, usually [50256])
-        const specials = new Set(tokenizer.all_special_ids || []);
-        const idsNoSpecials = ids.filter(id => !specials.has(id));
+        // Map special IDs -> special token strings (if available)
+        const specialIds = Array.from(tokenizer.all_special_ids || []);
+        const specialTokens = Array.from(tokenizer.all_special_tokens || []);
+        const idToSpecial = new Map(specialIds.map((id, i) => [id, specialTokens[i]]));
       
-        // Derive token strings from IDs
+        // Build token strings for every ID (specials included)
         let tokens = [];
         if (typeof tokenizer.convert_ids_to_tokens === 'function') {
-          tokens = tokenizer.convert_ids_to_tokens(idsNoSpecials);
+          tokens = tokenizer.convert_ids_to_tokens(ids);
         } else if (typeof tokenizer.id_to_token === 'function') {
-          tokens = idsNoSpecials.map(id => tokenizer.id_to_token(id));
-        } else if (!Array.isArray(enc)) {
-          // Some builds expose enc.tokens when enc is an object
-          const encTokens = Array.isArray(enc.tokens) ? enc.tokens : [];
-          tokens = encTokens.filter((_, i) => !specials.has(ids[i]));
+          tokens = ids.map(id => tokenizer.id_to_token(id));
+        } else if (!Array.isArray(enc) && Array.isArray(enc.tokens)) {
+          tokens = enc.tokens;
         } else {
-          // Last resort: stringify IDs (shouldn’t be needed with GPT-2)
-          tokens = idsNoSpecials.map(String);
+          // Fallback: decode each ID as a single-piece token
+          tokens = ids.map(id =>
+            tokenizer.decode([id], {
+              // we WANT specials in the stream; decode may return "" for them
+              skip_special_tokens: false,
+              clean_up_tokenization_spaces: false,
+            })
+          );
         }
       
+        // Ensure specials are visible: if a special token decodes to empty,
+        // replace it with its canonical name or a generic tag.
+        tokens = tokens.map((tok, i) => {
+          const id = ids[i];
+          if (tok && tok.length) return tok;
+          if (idToSpecial.has(id)) return idToSpecial.get(id); // e.g., <|endoftext|> for GPT-2
+          return `<special:${id}>`;
+        });
+      
         if (myRun !== runId) return;
       
         state.tokens = tokens;
-        state.ids = idsNoSpecials;
+        state.ids = ids; // include specials in the count
         render();
         status(`Done. ${state.tokens.length} tokens.`);
       } catch (e) {
@@ -190,25 +207,26 @@
     function render(){
       const tokens = Array.isArray(state.tokens) ? state.tokens : [];
       const ids    = Array.isArray(state.ids)    ? state.ids    : [];
-
-      // Tokens pane
+    
+      const specialSet = new Set(tokenizer.all_special_ids || []);
+    
       tokensEl.innerHTML = '';
       tokens.forEach((tok, i) => {
         const chip = document.createElement('span');
         chip.className = 'chip';
+        if (specialSet.has(ids[i])) chip.classList.add('special'); // <-- highlight specials
         chip.dataset.i = i;
         chip.textContent = tok;
         chip.addEventListener('mouseenter', ()=>highlight(i, true));
         chip.addEventListener('mouseleave', ()=>highlight(i, false));
         tokensEl.appendChild(chip);
       });
-
-      // IDs pane
+    
       idsEl.textContent = ids.join(' ');
-
       if (tokens.length === 0) status('Type to tokenize…');
     }
 
+
     function highlight(i, on){
       const ids = Array.isArray(state.ids) ? state.ids : [];
       if (!ids.length) return;