Fix Top 10 predictions display error

- Improve decode_token function with better error handling
- Add detailed logging for debugging token decoding issues
- Enhance build_byte_to_token_map with more informative error messages
- Add docstring to build_byte_to_token_map function

Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>

visualization/html_generator.py

@@ -182,17 +182,25 @@ def generate_comparison_html(
     """
 
     def decode_token(token_id: int, tokenizer, model_type: str) -> str:
+        """Decode a single token ID to text using the appropriate tokenizer."""
         if tokenizer is None:
             return f"[{token_id}]"
         try:
             if model_type in ["rwkv", "rwkv7"]:
-                return tokenizer.decode([token_id])
+                # RWKV tokenizer uses decode method
+                decoded = tokenizer.decode([token_id])
+                return decoded if decoded else f"[{token_id}]"
             else:
-                return tokenizer.decode([token_id])
-        except:
+                # HuggingFace tokenizer
+                decoded = tokenizer.decode([token_id])
+                return decoded if decoded else f"[{token_id}]"
+        except Exception as e:
+            print(f"Warning: Failed to decode token {token_id} ({model_type}): {e}")
             return f"[{token_id}]"
 
     def build_byte_to_token_map(text: str, tokenizer, model_type: str):
+        """Build mapping from byte position to token index using the correct tokenizer.
+        Returns a list of (start, end, token_idx) tuples for range-based lookup."""
         if tokenizer is None:
             return []
 
@@ -200,6 +208,7 @@ def generate_comparison_html(
 
         try:
             if model_type in ["rwkv", "rwkv7"]:
+                # RWKV tokenizer
                 tokenized = tokenizer.encode(text)
                 if hasattr(tokenized, "ids"):
                     token_ids = tokenized.ids
@@ -212,9 +221,11 @@ def generate_comparison_html(
                         token_bytes = tokenizer.decodeBytes([token_id])
                         token_ranges.append((byte_pos, byte_pos + len(token_bytes), idx))
                         byte_pos += len(token_bytes)
-                    except:
+                    except Exception as e:
+                        print(f"Warning: Failed to decode RWKV token {token_id}: {e}")
                         pass
             else:
+                # HuggingFace tokenizer - use TokenizerBytesConverter
                 tokenizer_name = getattr(tokenizer, "name_or_path", None)
                 if tokenizer_name:
                     converter = TokenizerBytesConverter(tokenizer_name, trust_remote_code=True)
@@ -223,6 +234,8 @@ def generate_comparison_html(
                     for idx, token_bytes in enumerate(token_bytes_list):
                         token_ranges.append((byte_pos, byte_pos + len(token_bytes), idx))
                         byte_pos += len(token_bytes)
+                else:
+                    print(f"Warning: Could not get tokenizer name for HF model")
         except Exception as e:
             print(f"Warning: Could not build byte-to-token map ({model_type}): {e}")
             return []