Fix JSON truncation by using base64 encoding

Replaced direct JSON embedding in HTML attributes with base64 encoding
to avoid issues with special characters breaking HTML attribute boundaries.
This completely eliminates the escaping problems that were causing the
"Unexpected end of JSON input" error in Top 10 predictions.

Changes:
- Encode topk JSON data as base64 before embedding in data attributes
- Decode base64 in JavaScript before parsing JSON
- Update debug logging to show base64 encoding status

Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>

visualization/html_generator.py

@@ -420,11 +420,13 @@ def generate_comparison_html(
                         pred[1],
                         [[tid, prob, decode_token(tid, tokenizer_a, model_type_a)] for tid, prob in pred[2]],
                     ]
-                    topk_a_json = json.dumps(decoded_pred, ensure_ascii=False)
+                    # Use base64 encoding to avoid escaping issues
+                    import base64
+                    topk_a_json = base64.b64encode(json.dumps(decoded_pred, ensure_ascii=False).encode('utf-8')).decode('ascii')
                     if token_count == 0:
-                        print(f"[DEBUG] Successfully generated topk_a_json")
-                        print(f"[DEBUG] Sample topk_a_json: {topk_a_json[:200]}")
-                        print(f"[DEBUG] After escape_for_attr: {escape_for_attr(topk_a_json)[:200]}")
+                        print(f"[DEBUG] Successfully generated topk_a_json (base64)")
+                        print(f"[DEBUG] Original JSON length: {len(json.dumps(decoded_pred, ensure_ascii=False))}")
+                        print(f"[DEBUG] Base64 length: {len(topk_a_json)}")
                 except Exception as e:
                     print(f"[DEBUG] Error generating topk_a_json at byte {byte_start}: {e}")
                     print(f"[DEBUG] pred[2] type: {type(pred[2])}")
@@ -439,10 +441,13 @@ def generate_comparison_html(
                     print(f"[DEBUG] pred structure: {pred}")
                 try:
                     decoded_pred = [pred[0], pred[1], [[tid, prob, decode_token(tid, tokenizer_b, model_type_b)] for tid, prob in pred[2]]]
-                    topk_b_json = json.dumps(decoded_pred, ensure_ascii=False)
+                    # Use base64 encoding to avoid escaping issues
+                    import base64
+                    topk_b_json = base64.b64encode(json.dumps(decoded_pred, ensure_ascii=False).encode('utf-8')).decode('ascii')
                     if token_count == 0:
-                        print(f"[DEBUG] Successfully generated topk_b_json")
-                        print(f"[DEBUG] Sample topk_b_json: {topk_b_json[:200]}")
+                        print(f"[DEBUG] Successfully generated topk_b_json (base64)")
+                        print(f"[DEBUG] Original JSON length: {len(json.dumps(decoded_pred, ensure_ascii=False))}")
+                        print(f"[DEBUG] Base64 length: {len(topk_b_json)}")
                 except Exception as e:
                     print(f"[DEBUG] Error generating topk_b_json at byte {byte_start}: {e}")
                     print(f"[DEBUG] pred[2] type: {type(pred[2])}")
@@ -823,9 +828,11 @@ def generate_comparison_html(
                 const top5A = token.getAttribute('data-topk-a') || '';
                 const top5B = token.getAttribute('data-topk-b') || '';
 
-                function formatTopkColumn(topkJson, modelName, titleClass) {{
-                    if (!topkJson) return '<div class="topk-column"><div class="topk-title ' + titleClass + '">' + modelName + '</div><div class="topk-list">N/A</div></div>';
+                function formatTopkColumn(topkBase64, modelName, titleClass) {{
+                    if (!topkBase64) return '<div class="topk-column"><div class="topk-title ' + titleClass + '">' + modelName + '</div><div class="topk-list">N/A</div></div>';
                     try {{
+                        // Decode base64 to JSON string
+                        const topkJson = atob(topkBase64);
                         const data = JSON.parse(topkJson);
                         const [actualId, rank, topkList] = data;
                         let html = '<div class="topk-column">';
@@ -851,7 +858,7 @@ def generate_comparison_html(
                         return html;
                     }} catch (e) {{
                         console.error('Error in formatTopkColumn for ' + modelName + ':', e);
-                        console.error('topkJson:', topkJson);
+                        console.error('topkBase64:', topkBase64);
                         return '<div class="topk-column"><div class="topk-title ' + titleClass + '">' + modelName + '</div><div class="topk-list">Error: ' + e.message + '</div></div>';
                     }}
                 }}