Escape html symbols in visualization

tests/test_utils.py

@@ -1,5 +1,6 @@
 import unittest
-from utils import load_hf_tokenizer, load_openai_tokenizer, load_tokenizers, tokenize, load_gsw_tokenizer
+from unittest.mock import patch
+from utils import load_hf_tokenizer, load_openai_tokenizer, load_tokenizers, tokenize, load_gsw_tokenizer, visualize_tokens
 
 class TestTokenizerLoading(unittest.TestCase):
     def test_load_hf_tokenizer(self):

utils.py

@@ -1,3 +1,4 @@
+import html
 from typing import Dict, List, Tuple
 from pathlib import Path
 
@@ -229,7 +230,8 @@ def visualize_tokens(text: str, tokenizers: Dict[str, object]):
         # Build the HTML with colored spans for each token
         for token in tokens:
             color = token_colors[token]
-            html += f'<span style="background-color: {color}; padding: 2px; margin: 1px; border-radius: 3px;">{token}</span>'
+            escaped_token = html.escape(token)
+            html += f'<span style="background-color: {color}; padding: 2px; margin: 1px; border-radius: 3px;">{escaped_token}</span>'
         
         results[name] = html