Update index.html

index.html

@@ -148,36 +148,36 @@
     
       status('Tokenizing…');
       try {
-        // 1) Get IDs (no options arg!)
-        const enc = await tokenizer.encode(text);
-
-        console.log('enc =', enc); // See what the tokenizer returns
-        // robustly pluck ids out of whatever shape the lib returns
-        let ids = (enc && (enc.ids ?? enc.input_ids ?? enc.inputIds)) || [];
-        console.log('ids=', ids, 'specials=', tokenizer.all_special_ids);
-        
-        // 2) Drop special tokens (e.g., BOS/EOS) for the demo
+        const enc = await tokenizer.encode(text); // returns EITHER an array OR an object, depending on tokenizer
+      
+        // >>> handle both shapes
+        let ids = Array.isArray(enc)
+          ? enc
+          : (enc && (enc.ids ?? enc.input_ids ?? enc.inputIds)) || [];
+      
+        // Drop special tokens (for GPT-2, usually [50256])
         const specials = new Set(tokenizer.all_special_ids || []);
         const idsNoSpecials = ids.filter(id => !specials.has(id));
-    
-        // 3) Turn IDs into token strings
+      
+        // Derive token strings from IDs
         let tokens = [];
         if (typeof tokenizer.convert_ids_to_tokens === 'function') {
           tokens = tokenizer.convert_ids_to_tokens(idsNoSpecials);
         } else if (typeof tokenizer.id_to_token === 'function') {
           tokens = idsNoSpecials.map(id => tokenizer.id_to_token(id));
-        } else if (Array.isArray(enc.tokens)) {
-          // fallback: some builds expose tokens directly (may include specials)
-          tokens = enc.tokens.filter((_, i) => !specials.has(ids[i]));
+        } else if (!Array.isArray(enc)) {
+          // Some builds expose enc.tokens when enc is an object
+          const encTokens = Array.isArray(enc.tokens) ? enc.tokens : [];
+          tokens = encTokens.filter((_, i) => !specials.has(ids[i]));
         } else {
+          // Last resort: stringify IDs (shouldn’t be needed with GPT-2)
           tokens = idsNoSpecials.map(String);
         }
-    
-        if (myRun !== runId) return; // drop stale result
-    
-        state.tokens = Array.isArray(tokens) ? tokens : [];
-        state.ids    = Array.isArray(idsNoSpecials) ? idsNoSpecials : [];
-    
+      
+        if (myRun !== runId) return;
+      
+        state.tokens = tokens;
+        state.ids = idsNoSpecials;
         render();
         status(`Done. ${state.tokens.length} tokens.`);
       } catch (e) {
@@ -185,7 +185,7 @@
         render();
         status('Error tokenizing. See console.');
       }
-    }
+
 
     function render(){
       const tokens = Array.isArray(state.tokens) ? state.tokens : [];