Update index.html

index.html

@@ -148,15 +148,33 @@
     
       status('Tokenizing…');
       try {
-        // Get token strings (robust, always works)
-        const tokens = await tokenizer.tokenize(text);
-    
-        // Get IDs (encode() may return different field names across versions)
-        const enc = await tokenizer.encode(text);
+        // 1) Always get IDs from encode
+        const enc = await tokenizer.encode(text, { add_special_tokens: false });
         const ids =
           (enc && (enc.ids ?? enc.input_ids ?? enc.inputIds)) || [];
     
-        if (myRun !== runId) return; // drop stale result
+        // 2) Derive token strings in a version-tolerant way
+        let tokens = [];
+    
+        // a) Preferred: convert_ids_to_tokens exists on many tokenizers
+        if (typeof tokenizer.convert_ids_to_tokens === 'function') {
+          tokens = tokenizer.convert_ids_to_tokens(ids);
+        }
+        // b) Fallback: id_to_token per-id
+        else if (typeof tokenizer.id_to_token === 'function') {
+          tokens = ids.map(id => tokenizer.id_to_token(id));
+        }
+        // c) Some builds include enc.tokens
+        else if (Array.isArray(enc.tokens)) {
+          tokens = enc.tokens;
+        }
+        // d) Last resort: stringify IDs (shouldn’t happen, but keeps UI stable)
+        else {
+          tokens = ids.map(String);
+        }
+    
+        if (myRun !== runId) return; // drop stale results
+    
         state.tokens = Array.isArray(tokens) ? tokens : [];
         state.ids    = Array.isArray(ids)    ? ids    : [];
     
@@ -169,7 +187,6 @@
       }
     }
 
-
     function render(){
       const tokens = Array.isArray(state.tokens) ? state.tokens : [];
       const ids    = Array.isArray(state.ids)    ? state.ids    : [];