统一可视化配置，并将信息密度单位从字符改为字节

client/src/index.html

@@ -85,7 +85,7 @@
             <section id="all_result" class="results-section">
                 <div id="stats" class="stats-container">
                     <div class="histogram-item">
-                        <div>token surprisal histogram</div>
+                        <div id="token_histogram_title"></div>
                         <svg id="stats_frac"></svg>
                     </div>
                     <div class="histogram-item">

client/src/ts/appInitializer.ts

@@ -6,7 +6,7 @@
 import * as d3 from 'd3';
 import { SimpleEventHandler } from './utils/SimpleEventHandler';
 import { TextAnalysisAPI } from './api/GLTR_API';
-import { getSurprisalColor } from './utils/SurprisalColorConfig';
+import { getTokenSurprisalColor, getByteSurprisalColor } from './utils/SurprisalColorConfig';
 
 /**
  * 公共初始化返回对象
@@ -14,8 +14,8 @@ import { getSurprisalColor } from './utils/SurprisalColorConfig';
 export interface CommonAppContext {
     eventHandler: SimpleEventHandler;
     api: TextAnalysisAPI;
-    surprisalColorScale: (value: number) => string;
-    textEncoder: TextEncoder;
+    tokenSurprisalColorScale: (value: number) => string;
+    byteSurprisalColorScale: (value: number) => string;
     totalSurprisalFormat: (n: number | null) => string;
 }
 
@@ -33,8 +33,8 @@ export function initializeCommonApp(apiPrefix: string = '', element?: Element):
     return {
         eventHandler: new SimpleEventHandler(targetElement),
         api: new TextAnalysisAPI(apiPrefix),
-        surprisalColorScale: getSurprisalColor,
-        textEncoder: new TextEncoder(),
+        tokenSurprisalColorScale: getTokenSurprisalColor,
+        byteSurprisalColorScale: getByteSurprisalColor,
         totalSurprisalFormat: (n: number | null) => n !== null && Number.isFinite(n) ? format(n) : String(n)
     };
 }

client/src/ts/compare.ts

@@ -40,6 +40,12 @@ import {ToolTip} from './vis/ToolTip';
 import { calculateHighlights } from './utils/highlightUtils';
 // 公共初始化模块
 import {initializeCommonApp} from './appInitializer';
+import {
+    TOKEN_SURPRISAL_HISTOGRAM_BASE,
+    BYTE_SURPRISAL_HISTOGRAM_BASE,
+    DELTA_BYTE_SURPRISAL_HISTOGRAM_BASE,
+    SURPRISAL_PROGRESS_BASE
+} from "./utils/visualizationConfigs";
 
 // 使用从 demoManager 导出的验证函数
 
@@ -114,7 +120,7 @@ type DemoColumnData = {
     lmfInstance?: GLTR_Text_Box;  // LMF实例引用（对比模式下使用）
     histograms: {
         stats_frac: Histogram | null;
-        stats_char_frac: Histogram | null;
+        stats_byte_frac: Histogram | null;
         stats_surprisal_progress: ScatterPlot | null;
     };
 };
@@ -123,7 +129,7 @@ window.onload = () => {
     // 初始化公共应用组件
     const api_prefix = URLHandler.parameters['api'] || '';
     const bodyElement = <Element>d3.select('body').node();
-    const { eventHandler, api, surprisalColorScale, textEncoder, totalSurprisalFormat } = initializeCommonApp(api_prefix, bodyElement);
+    const { eventHandler, api, tokenSurprisalColorScale, byteSurprisalColorScale, totalSurprisalFormat } = initializeCommonApp(api_prefix, bodyElement);
 
     const container = d3.select('#compare-container');
     const mainFrame = d3.select('.main_frame');
@@ -231,7 +237,7 @@ window.onload = () => {
         const metricsId = `text_metrics_${safeId}`;
         const errorId = `error_${safeId}`;
         const statsFracId = `stats_frac_${safeId}`;
-        const statsCharFracId = `stats_char_frac_${safeId}`;
+        const statsByteFracId = `stats_byte_frac_${safeId}`;
         const statsProgressId = `stats_surprisal_progress_${safeId}`;
         const textRenderId = `text_render_${safeId}`;
 
@@ -262,12 +268,12 @@ window.onload = () => {
                 </div>
                 <div id="${statsId}" class="stats" style="text-align:center;">
                     <div style="display:block;text-align: center;margin-bottom: 20px;">
-                        <div>token surprisal histogram</div>
+                        <div id="token_histogram_title_${safeId}"></div>
                         <svg id="${statsFracId}"></svg>
                     </div>
                     <div style="display:block;text-align: center;margin-bottom: 20px;">
-                        <div id="char_histogram_title_${safeId}">char surprisal histogram</div>
-                        <svg id="${statsCharFracId}"></svg>
+                        <div id="byte_histogram_title_${safeId}"></div>
+                        <svg id="${statsByteFracId}"></svg>
                     </div>
                     <div style="display:block;text-align: center;margin-bottom: 20px;">
                         <div>surprisal vs token progress</div>
@@ -282,7 +288,7 @@ window.onload = () => {
     // 处理单个 demo 的数据
     const processDemoData = (data: AnalysisData): FrontendAnalyzeResult => {
         const result = data.result;
-        const safeText = data.request?.text || '';
+        const safeText = data.request.text;
 
         // 验证数据
         if (!Array.isArray(result.bpe_strings) || result.bpe_strings.length === 0) {
@@ -327,7 +333,7 @@ window.onload = () => {
 
     // 为单个列渲染统计图表（使用ID）
     const renderStatsForColumn = (id: string, columnData: DemoColumnData) => {
-        if (!columnData.stats || !columnData.histograms.stats_frac || !columnData.histograms.stats_char_frac || !columnData.histograms.stats_surprisal_progress) {
+        if (!columnData.stats || !columnData.histograms.stats_frac || !columnData.histograms.stats_byte_frac || !columnData.histograms.stats_surprisal_progress) {
             return;
         }
 
@@ -336,68 +342,64 @@ window.onload = () => {
         const safeId = toSafeId(id);
 
         // 更新 token surprisal histogram（保持不变，不显示差分）
+        // 使用 19 个台阶，对应区间：[0,1), [1,2), ..., [17,18), [18,∞)
         columnData.histograms.stats_frac.update({
+            ...TOKEN_SURPRISAL_HISTOGRAM_BASE,
             data: stats.tokenSurprisals,
-            label: "surprisal",
-            no_bins: 20,
-            extent: [0, 20],
-            colorScale: surprisalColorScale,
+            colorScale: tokenSurprisalColorScale,
             averageValue: stats.tokenAverage ?? undefined,
-            averageLabel: 'bits/token'
         });
 
-        // 更新 char surprisal histogram（Diff列显示差分）
+        // 更新列视图中 token surprisal histogram 的标题文本
+        const tokenTitleElement = document.getElementById(`token_histogram_title_${safeId}`);
+        if (tokenTitleElement) {
+            tokenTitleElement.textContent = TOKEN_SURPRISAL_HISTOGRAM_BASE.label;
+        }
+
+        // 更新信息密度histogram（Diff列显示差分）
         if (isDiffColumn && columnData.diffStats) {
-            // Diff列：显示Δchar surprisal histogram
-            const deltaCharSurprisals = columnData.diffStats.deltaCharSurprisals;
+            // Diff列：显示Δ信息密度 histogram
+            const deltaByteSurprisals = columnData.diffStats.deltaByteSurprisals;
             
             // 计算平均差分
-            const deltaAverage = deltaCharSurprisals.length > 0
-                ? deltaCharSurprisals.reduce((sum, val) => sum + val, 0) / deltaCharSurprisals.length
+            const deltaAverage = deltaByteSurprisals.length > 0
+                ? deltaByteSurprisals.reduce((sum, val) => sum + val, 0) / deltaByteSurprisals.length
                 : 0;
             
-            const label = "Δchar surprisal histogram";
-            columnData.histograms.stats_char_frac.update({
-                data: deltaCharSurprisals,
-                label: label,
-                no_bins: 20,
-                extent: [-10, 10],
+            columnData.histograms.stats_byte_frac.update({
+                ...DELTA_BYTE_SURPRISAL_HISTOGRAM_BASE,
+                data: deltaByteSurprisals,
                 colorScale: getDiffColor,
                 averageValue: deltaAverage,
-                averageLabel: 'Δ bits/char'
             });
             
             // 更新标题文本
-            const titleElement = document.getElementById(`char_histogram_title_${safeId}`);
+            const titleElement = document.getElementById(`byte_histogram_title_${safeId}`);
             if (titleElement) {
-                titleElement.textContent = label;
+                titleElement.textContent = DELTA_BYTE_SURPRISAL_HISTOGRAM_BASE.label;
             }
         } else {
-            // Base列或非模型差分模式：显示原始char surprisal histogram
-            const label = "char surprisal histogram";
-            columnData.histograms.stats_char_frac.update({
-                data: stats.charSurprisals,
-                label: label,
-                no_bins: 20,
-                extent: [0, 20],
-                colorScale: surprisalColorScale,
-                averageValue: stats.charAverage ?? undefined,
-                averageLabel: 'bits/char'
+            // Base列或非模型差分模式：显示原始信息密度 histogram
+            // 使用 13 个台阶，对应区间：[0,0.5), [0.5,1), [1,1.5), ..., [5.5,6), [6,∞)
+            columnData.histograms.stats_byte_frac.update({
+                ...BYTE_SURPRISAL_HISTOGRAM_BASE,
+                data: stats.byteSurprisals,
+                colorScale: byteSurprisalColorScale,
+                averageValue: stats.byteAverage ?? undefined,
             });
             
             // 更新标题文本
-            const titleElement = document.getElementById(`char_histogram_title_${safeId}`);
+            const titleElement = document.getElementById(`byte_histogram_title_${safeId}`);
             if (titleElement) {
-                titleElement.textContent = label;
+                titleElement.textContent = BYTE_SURPRISAL_HISTOGRAM_BASE.label;
             }
         }
 
         // 更新 surprisal progress scatter plot（保持不变）
         if (stats.tokenSurprisals && stats.tokenSurprisals.length > 0) {
             columnData.histograms.stats_surprisal_progress.update({
+                ...SURPRISAL_PROGRESS_BASE,
                 data: stats.tokenSurprisals,
-                xLabel: 'token index',
-                yLabel: 'surprisal (bits)'
             });
         }
     };
@@ -537,8 +539,8 @@ window.onload = () => {
                 response = result.data;
             }
             const enhancedResult = processDemoData(response);
-            const safeText = response.request?.text || '';
-            const textStats = calculateTextStats(enhancedResult, safeText, textEncoder);
+            const safeText = response.request.text;
+            const textStats = calculateTextStats(enhancedResult, safeText);
 
             columnData.data = response;
             columnData.enhancedResult = enhancedResult;
@@ -551,7 +553,7 @@ window.onload = () => {
             showErrorForColumn(id, null);
 
             // 更新统计信息显示（从分析结果中获取实际使用的模型）
-            const resultModel = response.result?.model || null;
+            const resultModel = response.result.model;
             updateMetricsForColumn(id, textStats, resultModel);
 
             // 渲染统计图表
@@ -565,7 +567,7 @@ window.onload = () => {
                 // 重新渲染所有列的统计图表和指标（因为差分数据可能变化）
                 columnsData.forEach((colData, colId) => {
                     if (colData.stats) {
-                        const resultModel = colData.data?.result?.model || null;
+                        const resultModel = colData.data.result.model;
                         updateMetricsForColumn(colId, colData.stats, resultModel);
                         renderStatsForColumn(colId, colData);
                     }
@@ -577,7 +579,7 @@ window.onload = () => {
                     // 更新差分模式（因为差分数据可能变化）
                     const isDiffColumn = columnData.diffStats && !isBaseColumn(id);
                     if (isDiffColumn && columnData.diffStats) {
-                        columnData.lmfInstance.setDiffMode(true, columnData.diffStats.deltaCharSurprisals);
+                        columnData.lmfInstance.setDiffMode(true, columnData.diffStats.deltaByteSurprisals);
                     } else {
                         columnData.lmfInstance.setDiffMode(false, []);
                     }
@@ -606,7 +608,7 @@ window.onload = () => {
     const initializeColumnVisualizations = (id: string, columnData: DemoColumnData): void => {
         const safeId = toSafeId(id);
         const statsFracId = `#stats_frac_${safeId}`;
-        const statsCharFracId = `#stats_char_frac_${safeId}`;
+        const statsByteFracId = `#stats_byte_frac_${safeId}`;
         const statsProgressId = `#stats_surprisal_progress_${safeId}`;
 
         // 创建 Histogram 实例
@@ -616,8 +618,8 @@ window.onload = () => {
             { width: 400, height: 200 }
         );
 
-        columnData.histograms.stats_char_frac = new Histogram(
-            d3.select(statsCharFracId),
+        columnData.histograms.stats_byte_frac = new Histogram(
+            d3.select(statsByteFracId),
             eventHandler,
             { width: 400, height: 200 }
         );
@@ -669,7 +671,7 @@ window.onload = () => {
         // 设置差分模式（如果是Diff列）
         const isDiffColumn = modelDiffMode && columnData.diffStats && !isBaseColumn(id);
         if (isDiffColumn && columnData.diffStats) {
-            columnData.lmfInstance.setDiffMode(true, columnData.diffStats.deltaCharSurprisals);
+            columnData.lmfInstance.setDiffMode(true, columnData.diffStats.deltaByteSurprisals);
         } else {
             columnData.lmfInstance.setDiffMode(false, []);
         }
@@ -686,17 +688,17 @@ window.onload = () => {
     };
 
     // 根据 histogram source 解析出列的 safeId 和直方图类型
-    const parseHistogramSource = (source?: string): { safeId: string; histogramType: 'token' | 'char' } | null => {
+    const parseHistogramSource = (source?: string): { safeId: string; histogramType: 'token' | 'byte' } | null => {
         if (!source) {
             return null;
         }
 
-        const charPrefix = 'stats_char_frac';
+        const bytePrefix = 'stats_byte_frac';
         const tokenPrefix = 'stats_frac';
 
-        if (source.startsWith(charPrefix)) {
-            const safeId = source.substring(charPrefix.length).replace(/^_/, '');
-            return safeId ? { safeId, histogramType: 'char' } : null;
+        if (source.startsWith(bytePrefix)) {
+            const safeId = source.substring(bytePrefix.length).replace(/^_/, '');
+            return safeId ? { safeId, histogramType: 'byte' } : null;
         }
 
         if (source.startsWith(tokenPrefix)) {
@@ -736,12 +738,21 @@ window.onload = () => {
 
         const { columnData } = columnEntry;
 
-        // 仅在模型差分模式且文本渲染已初始化时处理高亮
-        if (!modelDiffMode || !columnData.lmfInstance) {
-            return;
+        // 在模型差分模式下，只有base列支持点击高亮
+        // 非差分模式下，仅在文本渲染已初始化时处理高亮
+        if (modelDiffMode) {
+            // 模型差分模式：只有base列支持点击高亮
+            if (!isBaseColumn(columnData.id) || !columnData.lmfInstance) {
+                return;
+            }
+        } else {
+            // 非模型差分模式：需要文本渲染已初始化
+            if (!columnData.lmfInstance) {
+                return;
+            }
         }
 
-        const { stats_frac, stats_char_frac } = columnData.histograms;
+        const { stats_frac, stats_byte_frac } = columnData.histograms;
 
         let enhancedResult = columnData.enhancedResult;
         if (!enhancedResult && columnData.data) {
@@ -756,21 +767,21 @@ window.onload = () => {
         // binIndex 为 -1 表示取消高亮
         if (ev.binIndex === -1) {
             stats_frac?.clearSelection();
-            stats_char_frac?.clearSelection();
+            stats_byte_frac?.clearSelection();
             columnData.lmfInstance.clearHighlight();
             return;
         }
 
         // 同一列内仅保持一个直方图的选中状态
-        if (parsed.histogramType === 'char') {
+        if (parsed.histogramType === 'byte') {
             stats_frac?.clearSelection();
         } else {
-            stats_char_frac?.clearSelection();
+            stats_byte_frac?.clearSelection();
         }
 
         // 使用通用的高亮计算函数
         const { x0, x1 } = ev;
-        const { indices, style } = calculateHighlights(parsed.histogramType, x0, x1, enhancedResult);
+        const { indices, style } = calculateHighlights(parsed.histogramType, x0, x1, ev.binIndex, ev.no_bins, enhancedResult);
         
         // 高亮这些 token
         columnData.lmfInstance.setHighlightedIndices(indices, style);
@@ -818,7 +829,7 @@ window.onload = () => {
             // 优先使用缓存的原文
             if (columnData.originalText !== undefined) {
                 text = columnData.originalText;
-            } else if (columnData.data?.request?.text !== undefined) {
+            } else if (columnData.data) {
                 text = columnData.data.request.text;
             }
             
@@ -894,7 +905,7 @@ window.onload = () => {
         columnsData.forEach((columnData, id) => {
             if (columnData.stats) {
                 // 更新统计信息显示
-                const resultModel = columnData.data?.result?.model || null;
+                const resultModel = columnData.data.result.model;
                 updateMetricsForColumn(id, columnData.stats, resultModel);
                 
                 // 重新渲染图表
@@ -918,7 +929,7 @@ window.onload = () => {
                     // 更新差分模式
                     const isDiffColumn = columnData.diffStats && !isBaseColumn(id);
                     if (isDiffColumn && columnData.diffStats) {
-                        columnData.lmfInstance.setDiffMode(true, columnData.diffStats.deltaCharSurprisals);
+                        columnData.lmfInstance.setDiffMode(true, columnData.diffStats.deltaByteSurprisals);
                     } else {
                         columnData.lmfInstance.setDiffMode(false, []);
                     }
@@ -946,7 +957,7 @@ window.onload = () => {
         columnsData.forEach((columnData, id) => {
             if (columnData.stats) {
                 // 更新统计信息显示
-                const resultModel = columnData.data?.result?.model || null;
+                const resultModel = columnData.data.result.model;
                 updateMetricsForColumn(id, columnData.stats, resultModel);
                 
                 // 重新渲染图表
@@ -1125,7 +1136,7 @@ window.onload = () => {
                     return;
                 }
                 
-                const preloadText = result.data.request?.text || '';
+                const preloadText = result.data.request.text;
                 
                 // 与已有 demo 的原文对比
                 const consistency = checkTextConsistency();
@@ -1161,7 +1172,7 @@ window.onload = () => {
             lmfInstance: undefined,
             histograms: {
                 stats_frac: null,
-                stats_char_frac: null,
+                stats_byte_frac: null,
                 stats_surprisal_progress: null
             }
         };
@@ -1537,7 +1548,7 @@ window.onload = () => {
             // 重新渲染所有列的统计图表和指标，并更新 LMF 实例的差分模式
             columnsData.forEach((columnData, id) => {
                 if (columnData.stats) {
-                    const resultModel = columnData.data?.result?.model || null;
+                    const resultModel = columnData.data.result.model;
                     updateMetricsForColumn(id, columnData.stats, resultModel);
                     renderStatsForColumn(id, columnData);
                 }
@@ -1546,7 +1557,7 @@ window.onload = () => {
                 if (columnData.lmfInstance) {
                     const isDiffColumn = columnData.diffStats && !isBaseColumn(id);
                     if (isDiffColumn && columnData.diffStats) {
-                        columnData.lmfInstance.setDiffMode(true, columnData.diffStats.deltaCharSurprisals);
+                        columnData.lmfInstance.setDiffMode(true, columnData.diffStats.deltaByteSurprisals);
                     } else {
                         columnData.lmfInstance.setDiffMode(false, []);
                     }
@@ -1694,7 +1705,7 @@ window.onload = () => {
             // 重新渲染所有列的统计图表和指标，并更新 LMF 实例的差分模式
             columnsData.forEach((columnData, id) => {
                 if (columnData.stats) {
-                    const resultModel = columnData.data?.result?.model || null;
+                    const resultModel = columnData.data.result.model;
                     updateMetricsForColumn(id, columnData.stats, resultModel);
                     renderStatsForColumn(id, columnData);
                 }
@@ -1703,7 +1714,7 @@ window.onload = () => {
                 if (columnData.lmfInstance) {
                     const isDiffColumn = columnData.diffStats && !isBaseColumn(id);
                     if (isDiffColumn && columnData.diffStats) {
-                        columnData.lmfInstance.setDiffMode(true, columnData.diffStats.deltaCharSurprisals);
+                        columnData.lmfInstance.setDiffMode(true, columnData.diffStats.deltaByteSurprisals);
                     } else {
                         columnData.lmfInstance.setDiffMode(false, []);
                     }

client/src/ts/controllers/highlightController.ts

@@ -39,11 +39,11 @@ export class HighlightController {
             return;
         }
 
-        const { x0, x1, source } = ev;
+        const { x0, x1, binIndex, no_bins, source } = ev;
         const data = currentData.result;
-        
+
         // 仅处理 token 直方图
-        const { indices, style } = calculateHighlights('token', x0, x1, data);
+        const { indices, style } = calculateHighlights('token', x0, x1, binIndex, no_bins, data);
         
         // 高亮这些 token
         this.options.lmf.setHighlightedIndices(indices, style);

client/src/ts/controllers/textInputController.ts

@@ -1,6 +1,7 @@
 import * as d3 from 'd3';
 import type { TextStats } from '../utils/textStatistics';
-import { calculateTextStats, getByteLength } from '../utils/textStatistics';
+import { calculateTextStats } from '../utils/textStatistics';
+import { countTokenCharacters } from '../utils/Util';
 import type { FrontendAnalyzeResult } from '../api/GLTR_API';
 
 /**
@@ -24,7 +25,6 @@ export type TextInputControllerOptions = {
     submitBtn: d3.Selection<any, unknown, any, any>;
     saveBtn: d3.Selection<any, unknown, any, any>;
     pasteBtn: d3.Selection<any, unknown, any, any>;
-    textEncoder: TextEncoder | null;
     totalSurprisalFormat: (value: number | null) => string;
     showAlertDialog: (title: string, message: string) => void;
 };
@@ -77,7 +77,7 @@ export class TextInputController {
         // 注意：submitBtn 的状态现在由外部状态系统统一管理，不再在这里设置
         
         if (!this.options.textCountValue.empty()) {
-            const charCount = Array.from(textValue).length;
+            const charCount = countTokenCharacters(textValue);
             this.options.textCountValue.text(charCount.toString());
         }
     }
@@ -205,9 +205,8 @@ export class TextInputController {
  */
 export const calculateTextStatsForController = (
     result: FrontendAnalyzeResult,
-    originalText: string,
-    textEncoder: TextEncoder | null
+    originalText: string
 ): TextStats => {
-    return calculateTextStats(result, originalText, textEncoder);
+    return calculateTextStats(result, originalText);
 };
 

client/src/ts/start.ts

@@ -61,7 +61,7 @@ window.onload = () => {
     // 初始化公共应用组件
     const api_prefix = URLHandler.parameters['api'] || '';
     const bodyElement = <Element>d3.select('body').node();
-    const { eventHandler, api, surprisalColorScale, textEncoder, totalSurprisalFormat } = initializeCommonApp(api_prefix, bodyElement);
+    const { eventHandler, api, tokenSurprisalColorScale, byteSurprisalColorScale, totalSurprisalFormat } = initializeCommonApp(api_prefix, bodyElement);
 
     // 页面初始化时确保 loading 状态被重置（防止刷新后仍显示转圈）
     d3.selectAll(".loadersmall").style('display', 'none');
@@ -115,7 +115,6 @@ window.onload = () => {
         submitBtn,
         saveBtn,
         pasteBtn,
-        textEncoder,
         totalSurprisalFormat,
         showAlertDialog
     });
@@ -176,8 +175,7 @@ window.onload = () => {
         stats_frac,
         stats_surprisal_progress,
         appStateManager,
-        textEncoder,
-        surprisalColorScale: surprisalColorScale as d3.ScaleSequential<string>
+        surprisalColorScale: tokenSurprisalColorScale as d3.ScaleSequential<string>
     });
 
     // 初始化主题管理器

client/src/ts/ui/dialog.ts

@@ -3,6 +3,7 @@
  * 提供统一的弹框样式和行为
  */
 import * as d3 from 'd3';
+import { countTokenCharacters } from '../utils/Util';
 
 export type DialogContentBuilder = (
     dialog: d3.Selection<HTMLDivElement, unknown, any, any>,
@@ -515,7 +516,7 @@ export function createNamePathTextContent(
         const updateTextCount = () => {
             const textNode = textarea.node() as HTMLTextAreaElement;
             const textValue = textNode?.value || '';
-            const charCount = Array.from(textValue).length;
+            const charCount = countTokenCharacters(textValue);
             textCountDisplay.text(`${charCount} 字`);
         };
 

client/src/ts/utils/SurprisalColorConfig.ts

@@ -3,51 +3,74 @@ import * as d3 from "d3";
 /**
  * 惊讶度颜色配置模块
  * 统一管理文本渲染和直方图的红色颜色配置
- * 20个台阶对应区间：[0,1), [1,2), ..., [18,19), [19,∞)
  */
 
+// ==========================================
+// 常量定义
+// ==========================================
+
+/** Token surprisal 的最大值，用于颜色映射 */
+const TOKEN_SURPRISAL_MAX = 18;
+
+/** Byte surprisal 的最大值，用于颜色映射 */
+const BYTE_SURPRISAL_MAX = 6;
+
+/** Minimap 颜色因子：用于放大颜色强度，因为平均后的byte surprisal密度会过小，需要放大以在minimap中更明显 */
+export const MINIMAP_COLOR_FACTOR = 1.3;
 
 /**
  * 根据归一化值获取对应的颜色（输入值应在[0,1]区间）
  * @param normalizedValue 归一化后的值，范围[0,1]
- * @param colorFactor 颜色因子，用于调整颜色强度，默认为0.7
  * @returns 颜色字符串（rgba格式，从透明到红色）
  */
-export function getSurprisalColorNormalized(normalizedValue: number, colorFactor: number = 0.7): string {
-    // 红色 #ff4740 对应的 RGB 值为 (255, 71, 64)
-    const startColor = "rgba(255, 71, 64, 0)"; // 完全透明的红色
-    const endColor = "rgba(255, 71, 64, 1)";   // 完全不透明的红色
-
-    // 使用d3的插值函数进行线性映射（支持rgba格式）
-    const colorInterpolator = d3.interpolate(startColor, endColor);
-
+export function getSurprisalColorNormalized(normalizedValue: number): string {
     // 确保输入值在[0,1]范围内
     const clampedValue = Math.max(0, Math.min(1, normalizedValue));
 
-    // 直接使用归一化值进行插值
-    return colorInterpolator(clampedValue * colorFactor);
-}
+    // 红色 #ff4740 对应的 RGB 值
+    const surprisalColor = "rgb(255, 71, 64)";
+    // 用于调整颜色强度，决定alpha范围[0, maxAlpha]
+    const maxAlpha = 0.7;
 
+    const alpha = clampedValue * maxAlpha;
+    return `rgba(255, 71, 64, ${alpha})`;
+}
 
 /**
- * 根据惊讶度值获取对应的颜色（线性映射，不取整）
- * @param surprisal 惊讶度值
- * @param colorFactor 颜色因子，用于调整颜色强度，默认为0.7
- * @returns 颜色字符串（rgba格式）
+ * 将惊讶度值线性映射到[0, 1]范围
+ * @param value 惊讶度值
+ * @param maxValue 最大值，value >= maxValue 时映射为1，value < 0 时映射为0
+ * @returns 归一化后的值，范围[0, 1]
  */
-export function getSurprisalColor(surprisal: number, colorFactor: number = 0.7): string {
-    // 将surprisal值映射到[0, 1]范围，其中0对应0，19及以上对应1
-    let normalizedValue: number;
-    if (surprisal < 0) {
-        normalizedValue = 0;
-    } else if (surprisal >= 19) {
-        normalizedValue = 1;
+function normalizeSurprisalValue(value: number, maxValue: number): number {
+    if (value < 0) {
+        return 0;
+    } else if (value >= maxValue) {
+        return 1;
     } else {
-        normalizedValue = surprisal / 19; // 线性映射到[0, 1]
+        return value / maxValue; // 线性映射到[0, 1]
     }
+}
+
+/**
+ * 根据token惊讶度值获取对应的颜色（线性映射，不取整）
+ * @param surprisal token惊讶度值，范围[0, TOKEN_SURPRISAL_MAX]会被映射到[0, 1]，超出范围会被截断
+ * @returns 颜色字符串（rgba格式）
+ */
+export function getTokenSurprisalColor(surprisal: number): string {
+    const normalizedValue = normalizeSurprisalValue(surprisal, TOKEN_SURPRISAL_MAX);
+    return getSurprisalColorNormalized(normalizedValue);
+}
 
-    // 调用归一化版本来计算颜色 
-    return getSurprisalColorNormalized(normalizedValue, colorFactor);
+/**
+ * 根据byte密度惊讶度值获取对应的颜色（线性映射，不取整）
+ * @param byteSurprisal byte密度惊讶度值，范围[0, BYTE_SURPRISAL_MAX]会被映射到[0, 1]，超出范围会被截断
+ * @param colorFactor 颜色因子，用于调整颜色强度，目前主要为了minimap显示更明显（平均后byte surprisal密度会过小，所以需要放大）。默认为1
+ * @returns 颜色字符串（rgba格式）
+ */
+export function getByteSurprisalColor(byteSurprisal: number, colorFactor: number = 1): string {
+    const normalizedValue = normalizeSurprisalValue(byteSurprisal * colorFactor, BYTE_SURPRISAL_MAX);
+    return getSurprisalColorNormalized(normalizedValue);
 }
 
 // ==========================================

client/src/ts/utils/Util.ts

@@ -1,4 +1,5 @@
 import * as d3 from "d3";
+import { TokenWithOffset } from "../api/generatedSchemas";
 
 /**
  * Created by hen on 5/15/17.
@@ -93,18 +94,64 @@ export function calculateSurprisal(probability: number): number {
  * @returns 字符数
  */
 export function countTokenCharacters(tokenText: string): number {
-    if (!tokenText) return 1; // 空token按1个字符处理，避免除零
     // 使用Array.from正确处理Unicode字符（包括中文、emoji等）
     return Array.from(tokenText).length;
 }
 
+// 复用 TextEncoder 实例，避免每次调用都创建新实例
+const textEncoder = new TextEncoder();
+
+/**
+ * 获取字符串的UTF-8编码字节长度
+ * @param value 要计算字节长度的字符串
+ * @returns UTF-8编码的字节数
+ */
+export const getByteLength = (value: string): number => {
+    return textEncoder.encode(value).length;
+};
+
 /**
- * 计算单位字符的surprisal值（惊讶度密度）
+ * 计算单位字节的surprisal值
  * @param surprisal token的总surprisal值
  * @param tokenText token文本
- * @returns 单位字符的surprisal值
+ * @returns 单位字节的surprisal值（bits/Byte）
  */
-export function calculateSurprisalPerCharacter(surprisal: number, tokenText: string): number {
-    const charCount = countTokenCharacters(tokenText);
-    return surprisal / charCount;
+function calculateSurprisalPerByte(surprisal: number, tokenText: string): number {
+    // 按UTF-8编码字节数计算
+    const byteCount = getByteLength(tokenText);
+    return byteCount > 0 ? surprisal / byteCount : 0;
+}
+
+/**
+ * 计算信息密度（统一接口，方便将来扩展）
+ * @param token token对象，包含real_topk和raw字段
+ * @returns 信息密度值（bits/Byte）
+ */
+export function calculateSurprisalDensity(token: TokenWithOffset): number {
+    const [rank, prob] = token.real_topk;
+    const surprisal = calculateSurprisal(prob);
+    const tokenText = token.raw;
+    return calculateSurprisalPerByte(surprisal, tokenText);
+}
+
+/**
+ * 为文本创建字符索引到字节索引的映射表
+ * @param text 原始文本
+ * @returns 数组，charToByteIndex[charIndex] = byteIndex
+ */
+export function buildCharToByteIndexMap(text: string): number[] {
+    const map: number[] = [];
+    let byteOffset = 0;
+    
+    for (let charIndex = 0; charIndex < text.length; charIndex++) {
+        map[charIndex] = byteOffset;
+        // 获取当前字符的UTF-8字节长度
+        const char = text[charIndex];
+        byteOffset += getByteLength(char);
+    }
+    
+    // 添加末尾位置（文本总字节长度）
+    map[text.length] = byteOffset;
+    
+    return map;
 }

client/src/ts/utils/demoPathUtils.ts

@@ -118,7 +118,7 @@ export const getDefaultDemoName = (
     }
     
     // 否则，使用第一行逻辑
-    const rawText = (currentData?.request?.text || textFieldValue || '').trim();
+    const rawText = (currentData ? currentData.request.text : textFieldValue || '').trim();
     if (!rawText) {
         return '新Demo';
     }

client/src/ts/utils/highlightUtils.ts

@@ -1,17 +1,21 @@
 import type { FrontendAnalyzeResult } from '../api/GLTR_API';
-import { calculateSurprisal, calculateSurprisalPerCharacter } from './Util';
+import { calculateSurprisal, calculateSurprisalDensity } from './Util';
 import { extractRealTopkFromTokens } from './tokenUtils';
 
 /**
  * 根据直方图 bin 的范围计算需要高亮的 token 索引集合（基于 token surprisal）
  * @param x0 bin 起始值
  * @param x1 bin 结束值
+ * @param binIndex bin在bins数组中的索引
+ * @param no_bins 直方图的总bin数量
  * @param result 前端分析结果（包含 originalTokens、mergedTokens、originalToMergedMap）
  * @returns 需要高亮的 merged token 索引集合
  */
 export function calculateTokenSurprisalHighlights(
     x0: number,
     x1: number,
+    binIndex: number,
+    no_bins: number,
     result: FrontendAnalyzeResult
 ): Set<number> {
     const highlightedIndices = new Set<number>();
@@ -20,13 +24,25 @@ export function calculateTokenSurprisalHighlights(
     const originalToMergedMap = result.originalToMergedMap;
     const mergedTokens = result.mergedTokens;
     
-    // 判断是否为最后一个 bin（[19, 20]）
-    const isLastBin = Math.abs(x0 - 19) < 0.001 && Math.abs(x1 - 20) < 0.001;
+    // 使用binIndex判断是否是最两侧的bin
+    const isFirstBin = binIndex === 0;  // 第一个bin：包含超出下界的值
+    const isLastBin = binIndex === no_bins - 1;  // 最后一个bin：包含超出上界的值
 
     // 遍历原始 token，找到 surprisal 在范围内的 token
     for (let i = 0; i < originalTokens.length; i++) {
-        const surprisal = calculateSurprisal(originalRealTopk[i]?.[1] ?? 0);
-        const inRange = isLastBin ? surprisal >= 19 : (surprisal >= x0 && surprisal < x1);
+        const surprisal = calculateSurprisal(originalRealTopk[i][1]);
+        let inRange = false;
+
+        if (isFirstBin) {
+            // 第一个bin：包含所有 < x1 的值（自身bin + 超出下界的数据）
+            inRange = surprisal < x1;
+        } else if (isLastBin) {
+            // 最后一个bin：包含所有 >= x0 的值（自身bin + 超出上界的数据）
+            inRange = surprisal >= x0;
+        } else {
+            // 中间bins：正常范围
+            inRange = surprisal >= x0 && surprisal < x1;
+        }
         
         if (inRange) {
             // 映射到 merged token 索引
@@ -41,30 +57,43 @@ export function calculateTokenSurprisalHighlights(
 }
 
 /**
- * 根据直方图 bin 的范围计算需要高亮的 token 索引集合（基于 char surprisal）
+ * 根据直方图 bin 的范围计算需要高亮的 token 索引集合（基于信息密度）
  * @param x0 bin 起始值
  * @param x1 bin 结束值
+ * @param binIndex bin在bins数组中的索引
+ * @param no_bins 直方图的总bin数量
  * @param result 前端分析结果（包含 mergedTokens）
  * @returns 需要高亮的 merged token 索引集合
  */
-export function calculateCharSurprisalHighlights(
+export function calculateByteSurprisalHighlights(
     x0: number,
     x1: number,
+    binIndex: number,
+    no_bins: number,
     result: FrontendAnalyzeResult
 ): Set<number> {
     const highlightedIndices = new Set<number>();
     const mergedTokens = result.mergedTokens;
-    const mergedRealTopk = extractRealTopkFromTokens(mergedTokens);
     
-    // 判断是否为最后一个 bin（[19, 20]）
-    const isLastBin = Math.abs(x0 - 19) < 0.001 && Math.abs(x1 - 20) < 0.001;
+    // 使用binIndex判断是否是最两侧的bin
+    const isFirstBin = binIndex === 0;  // 第一个bin：包含超出下界的值
+    const isLastBin = binIndex === no_bins - 1;  // 最后一个bin：包含超出上界的值
 
-    // 遍历 merged token，找到 char surprisal 在范围内的 token
+    // 遍历 merged token，找到信息密度在范围内的 token
     for (let i = 0; i < mergedTokens.length; i++) {
-        const surprisal = calculateSurprisal(mergedRealTopk[i]?.[1] ?? 0);
-        const tokenText = mergedTokens[i]?.raw || '';
-        const charSurprisal = calculateSurprisalPerCharacter(surprisal, tokenText);
-        const inRange = isLastBin ? charSurprisal >= 19 : (charSurprisal >= x0 && charSurprisal < x1);
+        const informationDensity = calculateSurprisalDensity(mergedTokens[i]);
+        let inRange = false;
+
+        if (isFirstBin) {
+            // 第一个bin：包含所有 < x1 的值（自身bin + 超出下界的数据）
+            inRange = informationDensity < x1;
+        } else if (isLastBin) {
+            // 最后一个bin：包含所有 >= x0 的值（自身bin + 超出上界的数据）
+            inRange = informationDensity >= x0;
+        } else {
+            // 中间bins：正常范围
+            inRange = informationDensity >= x0 && informationDensity < x1;
+        }
         
         if (inRange) {
             highlightedIndices.add(i);
@@ -77,13 +106,15 @@ export function calculateCharSurprisalHighlights(
 /**
  * 直方图类型
  */
-export type HistogramType = 'token' | 'char';
+export type HistogramType = 'token' | 'byte';
 
 /**
  * 根据直方图类型和 bin 范围计算需要高亮的 token 索引集合
- * @param histogramType 直方图类型（'token' 或 'char'）
+ * @param histogramType 直方图类型（'token' 或 'byte'）
  * @param x0 bin 起始值
  * @param x1 bin 结束值
+ * @param binIndex bin在bins数组中的索引
+ * @param no_bins 直方图的总bin数量
  * @param result 前端分析结果
  * @returns 需要高亮的 merged token 索引集合和对应的高亮样式
  */
@@ -91,16 +122,18 @@ export function calculateHighlights(
     histogramType: HistogramType,
     x0: number,
     x1: number,
+    binIndex: number,
+    no_bins: number,
     result: FrontendAnalyzeResult
 ): { indices: Set<number>; style: 'border' | 'underline' } {
-    if (histogramType === 'char') {
+    if (histogramType === 'byte') {
         return {
-            indices: calculateCharSurprisalHighlights(x0, x1, result),
+            indices: calculateByteSurprisalHighlights(x0, x1, binIndex, no_bins, result),
             style: 'underline'
         };
     } else {
         return {
-            indices: calculateTokenSurprisalHighlights(x0, x1, result),
+            indices: calculateTokenSurprisalHighlights(x0, x1, binIndex, no_bins, result),
             style: 'border'
         };
     }

client/src/ts/utils/localFileUtils.ts

@@ -76,7 +76,7 @@ export function validateDemoFormat(data: any): data is AnalysisData {
     }
 
     // 7. 验证token数据的一致性（offset和raw是否匹配）
-    const text = data.request?.text || '';
+    const text = data.request.text;
     if (text) {
         const consistencyError = validateTokenConsistency(
             data.result.bpe_strings as Array<{ offset?: [number, number]; raw?: string }>,

client/src/ts/utils/textStatistics.ts

@@ -1,5 +1,5 @@
 import type { FrontendAnalyzeResult } from '../api/GLTR_API';
-import { calculateSurprisal, calculateSurprisalPerCharacter, countTokenCharacters } from './Util';
+import { calculateSurprisal, calculateSurprisalDensity, countTokenCharacters, getByteLength } from './Util';
 import { extractRealTopkFromTokens } from './tokenUtils';
 
 export type TextStats = {
@@ -7,9 +7,9 @@ export type TextStats = {
     charCount: number;
     tokenCount: number;
     tokenSurprisals: number[];
-    charSurprisals: number[];
+    byteSurprisals: number[];
     tokenAverage: number | null;
-    charAverage: number | null;
+    byteAverage: number | null;
     totalSurprisal: number | null;
 };
 
@@ -25,7 +25,7 @@ export type DiffStats = {
     tokenAverage: number | null;
     // 差分字段
     deltaTotalSurprisal: number | null;  // Δ总surprisal
-    deltaCharSurprisals: number[];  // 逐字的Δ(bit/char)
+    deltaByteSurprisals: number[];  // 逐字节的Δ信息密度(bits/Byte)
 };
 
 /**
@@ -43,27 +43,12 @@ export const computeAverage = (values: number[] | null | undefined): number | nu
     return sum / validValues.length;
 };
 
-/**
- * 获取字节长度
- */
-export const getByteLength = (value: string, textEncoder: TextEncoder | null): number => {
-    if (textEncoder) {
-        return textEncoder.encode(value).length;
-    }
-    try {
-        return new Blob([value]).size;
-    } catch {
-        return value.length;
-    }
-};
-
 /**
  * 计算文本统计信息
  */
 export const calculateTextStats = (
     result: FrontendAnalyzeResult,
-    originalText: string,
-    textEncoder: TextEncoder | null
+    originalText: string
 ): TextStats => {
     const originalTokens = result.originalTokens;
     const mergedTokens = result.mergedTokens;
@@ -75,24 +60,24 @@ export const calculateTextStats = (
     let truncatedTextLength = 0;
     if (originalTokens.length > 0) {
         const lastToken = originalTokens[originalTokens.length - 1];
-        truncatedTextLength = lastToken?.offset?.[1] ?? 0;
+        truncatedTextLength = lastToken.offset[1];
     }
     
     // 从原始文本中截取实际分析的文本部分
     const truncatedText = originalText.slice(0, truncatedTextLength);
-    const safeText = truncatedText || '';
+    const safeText = truncatedText;
     
-    const byteCount = getByteLength(safeText, textEncoder);
-    const charCount = Array.from(safeText).length;
+    const byteCount = getByteLength(safeText);
+    const charCount = countTokenCharacters(safeText);
     const tokenCount = originalTokens.length;
 
     const tokenSurprisals: number[] = [];
-    const charSurprisals: number[] = [];
+    const byteSurprisals: number[] = [];
     let totalSurprisal = 0;
     let hasValidTotal = false;
 
     originalTokens.forEach((token, index) => {
-        const prob = realTopkOriginal[index]?.[1] ?? 0;
+        const prob = realTopkOriginal[index][1];
         const surprisal = calculateSurprisal(prob);
         tokenSurprisals.push(surprisal);
         if (Number.isFinite(surprisal)) {
@@ -101,14 +86,15 @@ export const calculateTextStats = (
         }
     });
 
-    mergedTokens.forEach((token, index) => {
-        const prob = realTopkMerged[index]?.[1] ?? 0;
-        const surprisal = calculateSurprisal(prob);
-        const tokenText = token?.raw || '';
-        const charCountForToken = countTokenCharacters(tokenText);
-        const charSurprisal = calculateSurprisalPerCharacter(surprisal, tokenText);
-        for (let i = 0; i < charCountForToken; i++) {
-            charSurprisals.push(charSurprisal);
+    mergedTokens.forEach((token) => {
+        const tokenText = token.raw;
+        const byteCountForToken = getByteLength(tokenText);
+        const byteSurprisal = calculateSurprisalDensity(token);
+        // 为token的每个字节添加相同的byteSurprisal值
+        // 注意：虽然可以使用Array.fill优化，但考虑到token的字节数通常很少（平均几个字节），
+        // 使用简单的循环更直观，性能差异可忽略不计
+        for (let i = 0; i < byteCountForToken; i++) {
+            byteSurprisals.push(byteSurprisal);
         }
     });
 
@@ -117,9 +103,9 @@ export const calculateTextStats = (
         charCount,
         tokenCount,
         tokenSurprisals,
-        charSurprisals,
+        byteSurprisals,
         tokenAverage: computeAverage(tokenSurprisals),
-        charAverage: computeAverage(charSurprisals),
+        byteAverage: computeAverage(byteSurprisals),
         totalSurprisal: hasValidTotal ? totalSurprisal : null
     };
 };
@@ -139,13 +125,13 @@ export const calculateDiffStats = (
         ? diffStats.totalSurprisal - baseStats.totalSurprisal
         : null;
 
-    // 计算逐字的Δ(bit/char)
-    const deltaCharSurprisals: number[] = [];
-    const minLength = Math.min(diffStats.charSurprisals.length, baseStats.charSurprisals.length);
+    // 计算逐字节的Δ信息密度(bits/Byte)
+    const deltaByteSurprisals: number[] = [];
+    const minLength = Math.min(diffStats.byteSurprisals.length, baseStats.byteSurprisals.length);
     
     for (let i = 0; i < minLength; i++) {
-        const delta = diffStats.charSurprisals[i] - baseStats.charSurprisals[i];
-        deltaCharSurprisals.push(delta);
+        const delta = diffStats.byteSurprisals[i] - baseStats.byteSurprisals[i];
+        deltaByteSurprisals.push(delta);
     }
 
     return {
@@ -155,7 +141,7 @@ export const calculateDiffStats = (
         tokenSurprisals: diffStats.tokenSurprisals,
         tokenAverage: diffStats.tokenAverage,
         deltaTotalSurprisal,
-        deltaCharSurprisals
+        deltaByteSurprisals
     };
 };
 

client/src/ts/utils/tokenUtils.ts

@@ -39,7 +39,7 @@ export const clonePredTopk = (list: [string, number][] | null | undefined): [str
 export const cloneFrontendToken = (token: FrontendToken, options: CloneTokenOptions = {}): FrontendToken => {
     const cloned: FrontendToken = {
         offset: [token.offset[0], token.offset[1]],
-        raw: token.raw ?? '',
+        raw: token.raw,
         real_topk: cloneRealTopk(token.real_topk),
         pred_topk: clonePredTopk(token.pred_topk)
     };
@@ -129,11 +129,8 @@ export const extractRealTopkFromTokens = (tokens: FrontendToken[] | null | undef
         return [];
     }
     return tokens.map((token) => {
-        const tuple = token?.real_topk;
-        if (Array.isArray(tuple) && tuple.length === 2 && tuple.every((item) => typeof item === 'number')) {
-            return [tuple[0], tuple[1]];
-        }
-        return [0, 0];
+        const tuple = token.real_topk;
+        return [tuple[0], tuple[1]];
     });
 };
 
@@ -142,7 +139,7 @@ export const extractRealTopkFromTokens = (tokens: FrontendToken[] | null | undef
  */
 export const createRawSnapshot = (response: AnalyzeResponse): AnalyzeResponse => {
     const requestClone: AnalyzeResponse['request'] = {
-        text: response.request?.text ?? ''
+        text: response.request.text
     };
     const originalResult = response.result;
     const tokensForSave = originalResult.bpe_strings.map((token) =>
@@ -150,7 +147,7 @@ export const createRawSnapshot = (response: AnalyzeResponse): AnalyzeResponse =>
     );
     // 确保 model 字段在最前面
     const resultClone: AnalyzeResponse['result'] = {
-        model: originalResult.model ?? null,
+        model: originalResult.model,
         ...originalResult,
         bpe_strings: tokensForSave
     };

client/src/ts/utils/visualizationConfigs.ts

@@ -0,0 +1,52 @@
+/**
+ * 直方图基础配置类型
+ */
+export interface HistogramBaseConfig {
+  label: string;
+  no_bins: number;
+  extent: [number, number];
+  averageLabel: string;
+  showLeftInfinity?: boolean;
+  xAxisTickSkip?: number;
+}
+
+/**
+ * 散点图基础配置类型
+ */
+export interface ScatterPlotBaseConfig {
+  xLabel: string;
+  yLabel: string;
+}
+
+// Token surprisal 直方图的公共外观配置（不包含数据和颜色）
+export const TOKEN_SURPRISAL_HISTOGRAM_BASE: HistogramBaseConfig = {
+  label: "token surprisal histogram",
+  no_bins: 19,
+  extent: [0, 19],
+  averageLabel: "bits/token",
+};
+
+// Byte surprisal 直方图（原始信息密度）的公共外观配置
+export const BYTE_SURPRISAL_HISTOGRAM_BASE: HistogramBaseConfig = {
+  label: "byte surprisal histogram",
+  no_bins: 13,
+  extent: [0, 6.5],
+  averageLabel: "bits/Byte",
+};
+
+// ΔByte surprisal 直方图（差分信息密度）的公共外观配置
+export const DELTA_BYTE_SURPRISAL_HISTOGRAM_BASE: HistogramBaseConfig = {
+  label: "Δbyte surprisal histogram",
+  no_bins: 20,
+  xAxisTickSkip: 1,  // x轴刻度数字绘制间隔
+  extent: [-5, 5],
+  averageLabel: "Δ bits/Byte",
+  showLeftInfinity: true,  // 左侧显示 -∞ 符号
+};
+
+// Surprisal progress 散点图的公共外观配置（不包含数据）
+export const SURPRISAL_PROGRESS_BASE: ScatterPlotBaseConfig = {
+  xLabel: "token index",
+  yLabel: "surprisal (bits)",
+} as const;
+

client/src/ts/utils/visualizationUpdater.ts

@@ -25,6 +25,7 @@ import {
     calculateTextStats,
     type TextStats
 } from './textStatistics';
+import { TOKEN_SURPRISAL_HISTOGRAM_BASE, SURPRISAL_PROGRESS_BASE } from "./visualizationConfigs";
 import { showAlertDialog } from '../ui/dialog';
 
 /**
@@ -37,7 +38,6 @@ export interface VisualizationDependencies {
     stats_frac: Histogram;
     stats_surprisal_progress: ScatterPlot;
     appStateManager: AppStateManager;
-    textEncoder: TextEncoder;
     surprisalColorScale: d3.ScaleSequential<string>;
 }
 
@@ -120,21 +120,24 @@ export class VisualizationUpdater {
         const currentTokenAvg = this.currentState.currentTokenAvg;
         
         if (currentSurprisals) {
+            // Token surprisal histogram: 使用 19 个台阶，对应区间：[0,1), [1,2), ..., [17,18), [18,∞)
             this.deps.stats_frac.update({
+                ...TOKEN_SURPRISAL_HISTOGRAM_BASE,
                 data: currentSurprisals,
-                label: "surprisal",
-                no_bins: 20,
-                extent: [0, 20],
                 colorScale: this.deps.surprisalColorScale,
                 averageValue: currentTokenAvg ?? undefined,
-                averageLabel: 'bits/token'
             });
+
+            // 更新主视图中 token surprisal histogram 的标题文本
+            const titleElement = document.getElementById('token_histogram_title');
+            if (titleElement) {
+                titleElement.textContent = TOKEN_SURPRISAL_HISTOGRAM_BASE.label;
+            }
         }
         if (currentSurprisals && currentSurprisals.length > 0) {
             this.deps.stats_surprisal_progress.update({
+                ...SURPRISAL_PROGRESS_BASE,
                 data: currentSurprisals,
-                xLabel: 'token index',
-                yLabel: 'surprisal (bits)'
             });
         }
     }
@@ -204,7 +207,7 @@ export class VisualizationUpdater {
                 return;
             }
 
-            const safeText = data.request?.text || '';
+            const safeText = data.request.text;
             const validationError = validateTokenConsistency(result.bpe_strings, safeText, { allowOverlap: true });
             if (validationError) {
                 abortDueToInvalidResponse(validationError);
@@ -245,7 +248,7 @@ export class VisualizationUpdater {
             // 只调用 lmf.update，不调用任何统计更新
             this.deps.lmf.update(enhancedResult);
 
-            const textStats = calculateTextStats(enhancedResult, safeText, this.deps.textEncoder);
+            const textStats = calculateTextStats(enhancedResult, safeText);
 
             // 保存当前surprisal数据，用于主题切换时重新渲染
             this.currentState.currentSurprisals = textStats.tokenSurprisals;
@@ -253,14 +256,14 @@ export class VisualizationUpdater {
             this.currentState.currentTotalSurprisal = textStats.totalSurprisal;
 
             // 更新文本指标和模型显示（从分析结果中获取实际使用的模型）
-            const resultModel = data.result?.model || null;
+            const resultModel = data.result.model;
             this.updateTextMetrics(textStats, resultModel);
 
             // Analyze 渲染完成后关闭动画，避免拖拽等二次渲染再次播放
             if (!disableAnimation) {
                 // 延迟关闭，确保动画有足够时间完成
                 // 动画时长估算：初始延迟100ms + 批次处理时间（根据token数量）
-                const tokenCount = enhancedResult.bpe_strings?.length || 0;
+                const tokenCount = enhancedResult.bpe_strings.length;
                 const estimatedAnimationTime = 100 + Math.ceil(tokenCount / 50) * 100;
                 const delayTime = Math.max(2000, estimatedAnimationTime + 500);
 

client/src/ts/vis/GLTR_Text_Box.ts

@@ -1,10 +1,9 @@
 import {VComponent} from "./VisComponent";
 import {FrontendAnalyzeResult} from "../api/GLTR_API";
-import {D3Sel, calculateSurprisal, calculateSurprisalPerCharacter} from "../utils/Util";
+import {D3Sel, calculateSurprisal, calculateSurprisalDensity, buildCharToByteIndexMap} from "../utils/Util";
 import {SimpleEventHandler} from "../utils/SimpleEventHandler";
 import * as d3 from "d3";
 import {RenderAnimator, TokenRenderTask} from "./RenderAnimator";
-import {getSurprisalColor} from "../utils/SurprisalColorConfig";
 import {HighlightManager} from "./HighlightManager";
 import {SvgOverlayManager} from "./SvgOverlayManager";
 import {TokenPositionCalculator} from "./TokenPositionCalculator";
@@ -42,11 +41,7 @@ export enum GLTR_Mode {
 }
 
 export type GLTR_RenderItem = {
-    top: number;
-    prop: number;
-    others: [string, number][];
-    token: string;
-    bpeMerged?: boolean;
+    tokenData: import('../api/GLTR_API').FrontendToken; // 完整的token对象，包含所有信息
 };
 export type GLTR_HoverEvent = { hovered: boolean, d: GLTR_RenderItem, event?: MouseEvent }
 
@@ -57,7 +52,8 @@ export class GLTR_Text_Box extends VComponent<FrontendAnalyzeResult> {
         highlightStyle: 'border' as 'border' | 'underline',  // 当前高亮样式
         // 差分渲染相关
         diffMode: false,  // 是否启用差分渲染模式
-        deltaCharSurprisals: [] as number[],  // 逐字的Δ(bit/char)数据
+        deltaByteSurprisals: [] as number[],  // 逐字节的Δ信息密度(bits/Byte)
+        charToByteIndexMap: [] as number[],  // 字符索引到字节索引的映射表
     };
     protected css_name = "LMF";
     protected options = {
@@ -172,6 +168,12 @@ export class GLTR_Text_Box extends VComponent<FrontendAnalyzeResult> {
         // 保存当前渲染数据
         this.currentRenderData = rd;
 
+        // 如果差分模式已启用，更新字符到字节的映射表（使用最新的原始文本）
+        if (this._current.diffMode && this._current.deltaByteSurprisals.length > 0) {
+            const originalText = rd.originalText;
+            this._current.charToByteIndexMap = buildCharToByteIndexMap(originalText);
+        }
+
         // 隐藏加载状态
         this.hideLoading();
 
@@ -221,7 +223,8 @@ export class GLTR_Text_Box extends VComponent<FrontendAnalyzeResult> {
                 getTokenRealTopk: (rd, tokenIndex) => this.getTokenRealTopk(rd, tokenIndex),
                 addTokenEventListeners: (element, tokenIndex, rd) => this.addTokenEventListeners(element, tokenIndex, rd),
                 diffMode: this._current.diffMode,
-                deltaCharSurprisals: this._current.deltaCharSurprisals
+                deltaByteSurprisals: this._current.deltaByteSurprisals,
+                charToByteIndexMap: this._current.charToByteIndexMap
             });
         } else {
             // 更新差分模式和数据
@@ -229,7 +232,8 @@ export class GLTR_Text_Box extends VComponent<FrontendAnalyzeResult> {
                 getTokenRealTopk: (rd, tokenIndex) => this.getTokenRealTopk(rd, tokenIndex),
                 addTokenEventListeners: (element, tokenIndex, rd) => this.addTokenEventListeners(element, tokenIndex, rd),
                 diffMode: this._current.diffMode,
-                deltaCharSurprisals: this._current.deltaCharSurprisals
+                deltaByteSurprisals: this._current.deltaByteSurprisals,
+                charToByteIndexMap: this._current.charToByteIndexMap
             });
         }
 
@@ -333,7 +337,7 @@ export class GLTR_Text_Box extends VComponent<FrontendAnalyzeResult> {
         }
 
         // originalText 始终由前端注入，直接使用（更接近用户输入）
-        const fullText = rd.originalText || '';
+        const fullText = rd.originalText;
 
         // 创建一个文本容器div，确保文本在SVG上方
         if (fullText) {
@@ -598,8 +602,8 @@ export class GLTR_Text_Box extends VComponent<FrontendAnalyzeResult> {
      * 获取指定token的真实概率信息
      */
     private getTokenRealTopk(rd: FrontendAnalyzeResult, tokenIndex: number): [number, number] | undefined {
-        const token = rd?.bpe_strings?.[tokenIndex];
-        return token && Array.isArray(token.real_topk)
+        const token = rd.bpe_strings[tokenIndex];
+        return token.real_topk
             ? token.real_topk as [number, number]
             : undefined;
     }
@@ -610,23 +614,13 @@ export class GLTR_Text_Box extends VComponent<FrontendAnalyzeResult> {
      */
     private addTokenEventListeners(element: SVGGElement, tokenIndex: number, rd: FrontendAnalyzeResult): void {
         const tokenData = rd.bpe_strings[tokenIndex];
-        const topK = this.getTokenRealTopk(rd, tokenIndex);
-        const tokenPredTopK = Array.isArray(tokenData?.pred_topk)
-            ? tokenData.pred_topk as [string, number][]
-            : [];
-        const isMerged = tokenData?.bpe_merged === true;
-        const tokenText = tokenData?.raw || '';
-
-            // 创建事件处理函数
+
+        // 创建事件处理函数
         const handleMouseEnter = (event: MouseEvent) => {
             this.eventHandler.trigger(GLTR_Text_Box.events.tokenHovered, <GLTR_HoverEvent>{
                 hovered: true,
                 d: {
-                    token: tokenText,
-                    top: topK?.[0] || 0,
-                    prop: topK?.[1] || 0,
-                    others: tokenPredTopK,
-                    bpeMerged: isMerged
+                    tokenData: tokenData
                 },
                 event: event
             });
@@ -643,11 +637,7 @@ export class GLTR_Text_Box extends VComponent<FrontendAnalyzeResult> {
             this.eventHandler.trigger(GLTR_Text_Box.events.tokenHovered, <GLTR_HoverEvent>{
                 hovered: false,
                 d: {
-                    token: tokenText,
-                    top: topK?.[0] || 0,
-                    prop: topK?.[1] || 0,
-                    others: tokenPredTopK,
-                    bpeMerged: isMerged
+                    tokenData: tokenData
                 },
                 event: event
             });
@@ -771,9 +761,9 @@ export class GLTR_Text_Box extends VComponent<FrontendAnalyzeResult> {
     }
 
     protected _wrangle(data: FrontendAnalyzeResult) {
-        const tokens = Array.isArray(data?.bpe_strings) ? data.bpe_strings : [];
+        const tokens = data.bpe_strings;
         const allTop1 = tokens
-            .map(token => Array.isArray(token?.pred_topk) && token.pred_topk.length > 0 ? token.pred_topk[0][1] : null)
+            .map(token => token.pred_topk.length > 0 ? token.pred_topk[0][1] : null)
             .filter((value): value is number => typeof value === 'number' && Number.isFinite(value));
 
         if (allTop1.length === 0) {
@@ -862,14 +852,19 @@ export class GLTR_Text_Box extends VComponent<FrontendAnalyzeResult> {
     /**
      * 设置差分渲染模式和数据
      * @param enabled 是否启用差分模式
-     * @param deltaCharSurprisals 逐字的Δ(bit/char)数据
+     * @param deltaByteSurprisals 逐字节的Δ信息密度(bits/Byte)
      */
-    setDiffMode(enabled: boolean, deltaCharSurprisals: number[] = []) {
+    setDiffMode(enabled: boolean, deltaByteSurprisals: number[] = []) {
         this._current.diffMode = enabled;
-        this._current.deltaCharSurprisals = deltaCharSurprisals;
+        this._current.deltaByteSurprisals = deltaByteSurprisals;
         
-        // 如果有当前渲染数据，重新渲染（无论是开启还是关闭差分模式都需要重新渲染）
+        // 如果有当前渲染数据，构建字符索引到字节索引的映射表并重新渲染
         if (this.currentRenderData) {
+            // 构建字符索引到字节索引的映射表
+            // 使用当前渲染数据的原始文本
+            const originalText = this.currentRenderData.originalText;
+            this._current.charToByteIndexMap = buildCharToByteIndexMap(originalText);
+            
             // 差分模式切换时禁用动画
             const originalAnimationSetting = this.options.enableRenderAnimation;
             this.options.enableRenderAnimation = false;

client/src/ts/vis/Histogram.ts

@@ -9,11 +9,13 @@ const averageNumberFormat = d3.format('.2f');
 export type HistogramData = {
     data: number[],
     label?: string,
-    no_bins?: number,
-    extent?: number[],
-    colorScale?: (value: number) => string,  // 添加颜色 scale
+    no_bins: number,
+    extent: number[],
+    colorScale: (value: number) => string,  // 添加颜色 scale
     averageValue?: number,
-    averageLabel?: string
+    averageLabel?: string,
+    showLeftInfinity?: boolean,  // 是否在左侧显示 -∞ 符号
+    xAxisTickSkip?: number  // x轴刻度数字绘制间隔，0表示不跳过，1表示隔一个绘制一个（0,2,4...）
 }
 
 
@@ -22,6 +24,7 @@ export type HistogramBinClickEvent = {
     x0: number;
     x1: number;
     data: number[];
+    no_bins: number;   // 直方图的bin数量
     source?: string;  // 直方图标识，用于区分不同的直方图实例
 }
 
@@ -81,8 +84,26 @@ export class Histogram extends VComponent<HistogramData> {
     protected _render(rD: HistogramData): void {
         const op = this.options;
 
-        const values = rD.data.map(d => +d).filter(d => isFinite(d)).sort((a, b) => a - b);
-        
+        // extent 是必选参数，直接使用
+        const extent = rD.extent;
+
+        // 计算bin宽度
+        const binWidth = (extent[1] - extent[0]) / rD.no_bins;
+
+        // 超出上下界的按照对应bin的中心值处理
+        const values = rD.data.map(d => +d)
+            .filter(d => isFinite(d))
+            .map(d => {
+                if (d >= extent[1]) {
+                    // 超出或等于上界：映射到最后一个bin的中心值，避免d3.bin()为等于extent[1]的值创建额外的[19,19]bin
+                    return extent[1] - 0.5 * binWidth;
+                } else if (d <= extent[0]) {
+                    // 超出或等于下界：映射到第一个bin的中心值，避免d3.bin()为等于extent[0]的值创建额外的[0,0]bin
+                    return extent[0] + 0.5 * binWidth;
+                }
+                return d;
+            });
+
         // 如果数据为空，显示空图表
         if (values.length === 0) {
             this.layers.main.selectAll('.bar').remove();
@@ -91,31 +112,13 @@ export class Histogram extends VComponent<HistogramData> {
             this.layers.fg.selectAll('.avg-label').remove();
             return;
         }
-        
-        let extent = rD.extent || d3.extent(values);
-        
-        // 安全检查：确保 extent 有效
-        // 检查 extent 是否为 null/undefined、长度不为2、包含非有限值、或范围无效（包括 [0, 0] 这种情况）
-        if (!extent || extent.length !== 2 || !isFinite(extent[0]) || !isFinite(extent[1]) || extent[0] >= extent[1]) {
-            // 如果 extent 无效，使用数据的实际范围，如果数据也为空则使用默认值 [0, 1]
-            if (values.length > 0) {
-                const dataExtent = d3.extent(values);
-                if (dataExtent && dataExtent.length === 2 && isFinite(dataExtent[0]) && isFinite(dataExtent[1]) && dataExtent[0] < dataExtent[1]) {
-                    extent = dataExtent;
-                } else {
-                    extent = [0, 1];
-                }
-            } else {
-                extent = [0, 1];
-            }
-        }
-        
+
         // 如果指定了 extent，确保使用 extent 作为 domain，而不是 nice() 调整后的 domain
         // 这样可以保证 extent 的上限被正确使用，即使数据被截断了
         // 使用 extent 作为 domain，确保范围正确
         const padding = { left: 24, right: 35 };
         let valueScale = d3.scaleLinear().domain([extent[0], extent[1]]).range([padding.left, op.width - padding.right]);
-        
+
         const hasAverageValue = typeof rD.averageValue === 'number' && Number.isFinite(rD.averageValue);
         const clampedAverage = hasAverageValue
             ? Math.min(Math.max(rD.averageValue as number, extent[0]), extent[1])
@@ -123,26 +126,13 @@ export class Histogram extends VComponent<HistogramData> {
         const averageX = hasAverageValue && clampedAverage !== null
             ? valueScale(clampedAverage)
             : null;
-        
-        // 如果指定了 no_bins=20 且 extent=[0, 20]，使用固定的20个台阶阈值
-        // 阈值正好是 [0, 1, 2, ..., 19, 20]，对应区间 [0,1), [1,2), ..., [18,19), [19,20]
-        // 如果指定了 no_bins=20 且 extent=[-10, 10]（差分模式），使用固定的20个台阶阈值
-        // 阈值正好是 [-10, -9, -8, ..., 8, 9, 10]，对应区间 [-10,-9), [-9,-8), ..., [8,9), [9,10]
-        let thresholds: number[];
-        if (rD.no_bins === 20 && extent[0] === 0 && extent[1] === 20) {
-            // 固定20个台阶：生成阈值 [0, 1, 2, ..., 19, 20]
-            thresholds = Array.from({ length: 21 }, (_, i) => i); // [0, 1, 2, ..., 20]
-        } else if (rD.no_bins === 20 && extent[0] === -10 && extent[1] === 10) {
-            // 差分模式：生成阈值 [-10, -9, -8, ..., 8, 9, 10]
-            thresholds = Array.from({ length: 21 }, (_, i) => i - 10); // [-10, -9, ..., 9, 10]
-        } else {
-            // 其他情况使用原来的逻辑
-            const idealNoBins = rD.no_bins || Math.min(d3.thresholdFreedmanDiaconis(values, extent[0], extent[1]), 20);
-            thresholds = d3.ticks(extent[0], extent[1], idealNoBins);
-            // 确保最后一个阈值小于 extent[1]，这样最后一个 bin 可以包含所有 >= 最后一个阈值的数据
-            thresholds = thresholds.filter(t => t < extent[1]);
-        }
 
+        // 统一的阈值生成逻辑：生成有限数阈值，两侧bin自动包含超出范围的值
+        // no_bins 是必选参数，直接使用等宽bin
+        // thresholds 长度应该是 no_bins - 1，从 extent[0]+binWidth 开始，比如[0,10]，10个bin，则thresholds长度为9，分别是1..9，不包括0和10
+        const thresholds = Array.from({ length: rD.no_bins - 1 }, (_, i) => extent[0] + (i + 1) * binWidth);
+
+        // 设置domain确保边界严格按照extent划分，而不是实际的数据最大值和最小值
         const histo = d3.bin()
             .domain(<[number, number]>[extent[0], extent[1]])
             .thresholds(thresholds)(values);
@@ -200,22 +190,9 @@ export class Histogram extends VComponent<HistogramData> {
                 },
             })
             .style('fill', d => {
-                if (rD.colorScale) {
-                    // 判断是否是差分模式（extent 包含负值）
-                    const isDiffMode = rD.extent && rD.extent[0] < 0;
-                    
-                    if (isDiffMode) {
-                        // 差分模式：负值区间使用左边界 x0，非负值区间使用右边界 x1
-                        // 例如：bin [-1,0) 使用 x0=-1，bin [0,1) 使用 x1=1
-                        const colorValue = d.x0 < 0 ? d.x0 : d.x1;
-                        return rD.colorScale(colorValue);
-                    } else {
-                        // 正常模式：使用 bin 的起始值 x0 计算颜色，以匹配区间定义
-                        // 例如：bin [0,1) 使用 x0=0，bin [1,2) 使用 x0=1，bin [19,20) 使用 x0=19
-                        return rD.colorScale(d.x0);
-                    }
-                }
-                return '#666'; // 默认颜色
+                // 统一使用bin的中间值计算颜色
+                const colorValue = (d.x0 + d.x1) / 2;
+                return rD.colorScale(colorValue);
             })
             .style('stroke', (d, i) => {
                 // 如果这个bin被选中，添加蓝色边框
@@ -319,6 +296,7 @@ export class Histogram extends VComponent<HistogramData> {
                         x0: d.x0,
                         x1: d.x1,
                         data: d,
+                        no_bins: rD.no_bins,
                         source: sourceId
                     });
                 } else {
@@ -334,6 +312,7 @@ export class Histogram extends VComponent<HistogramData> {
                         x0: d.x0,
                         x1: d.x1,
                         data: d,
+                        no_bins: rD.no_bins,
                         source: sourceId
                     });
                 }
@@ -342,21 +321,41 @@ export class Histogram extends VComponent<HistogramData> {
 
         this.layers.bg.select('.y-axis').call(<any>d3.axisRight(countScale).tickFormat(op.numberFormat));
         
-        // Custom tick format: replace the minimum extent value with -∞ and maximum with ∞
+        // 构建所有刻度值数组，用于确定索引位置
+        const allTickValues = [extent[0], ...thresholds, extent[1]];  // 包含边界和所有阈值
+        const tickSkip = rD.xAxisTickSkip ?? 0;
+        
+        // Custom tick format: 左侧只有配置了 showLeftInfinity 的直方图显示 -∞，右侧所有直方图都显示 ∞
+        // 根据 xAxisTickSkip 参数决定是否显示数字标签（刻度线始终显示）
         const xAxisTickFormat = (d: number) => {
+            // 查找当前刻度值在数组中的索引
+            const tickIndex = allTickValues.findIndex(tick => Math.abs(tick - d) < 0.001);
+            
+            // 如果配置了 xAxisTickSkip，根据索引决定是否显示标签
+            if (tickSkip > 0 && tickIndex >= 0) {
+                // 如果索引不符合跳过规则，返回空字符串（不显示数字，但刻度线仍会显示）
+                if (tickIndex % (tickSkip + 1) !== 0) {
+                    return '';
+                }
+            }
+            
             if (rD.extent) {
-                // 检查是否是最小值（通常是 -10，差分模式的左边界）
-                if (Math.abs(d - rD.extent[0]) < 0.001) {
+                // 检查是否是最小值，且配置了显示左侧无穷大符号
+                if (rD.showLeftInfinity && Math.abs(d - rD.extent[0]) < 0.001) {
                     return '-∞';
                 }
-                // 检查是否是最大值（通常是 10 或 20，��边界）
+                // 检查是否是最大值，所有直方图右侧都显示 ∞
                 if (Math.abs(d - rD.extent[1]) < 0.001) {
                     return '∞';
                 }
             }
             return op.numberFormat(d);
         };
-        this.layers.bg.select('.x-axis').call(<any>d3.axisBottom(valueScale).tickFormat(xAxisTickFormat).ticks(thresholds.length));
+        
+        const xAxis = d3.axisBottom(valueScale)
+            .tickFormat(xAxisTickFormat)
+            .tickValues(allTickValues);
+        this.layers.bg.select('.x-axis').call(<any>xAxis);
 
 
     }

client/src/ts/vis/ScrollbarMinimap.ts

@@ -1,5 +1,6 @@
-import { getSurprisalColor } from '../utils/SurprisalColorConfig';
+import { getByteSurprisalColor, MINIMAP_COLOR_FACTOR } from '../utils/SurprisalColorConfig';
 import { isNarrowScreen } from '../utils/responsive';
+import { calculateSurprisalDensity } from '../utils/Util';
 import type { TokenFragmentRect } from './types';
 import type { FrontendAnalyzeResult } from '../api/GLTR_API';
 
@@ -22,7 +23,7 @@ interface AggregationResult {
  */
 interface BucketData {
     y: number;
-    surprisalSum: number;
+    surprisalDensitySum: number;
     TokenFragmentCount: number;
 }
 
@@ -210,11 +211,11 @@ export class ScrollbarMinimap {
         const { buckets } = aggregationResult;
 
         buckets.forEach(bucket => {
-            // 计算平均surprisal：总surprisal（surprisalPerChar累加）除以token数
-            const averageSurprisal = bucket.TokenFragmentCount > 0
-                ? bucket.surprisalSum / bucket.TokenFragmentCount
+            // 计算平均surprisal密度：总surprisal（surprisalPerByte累加）除以token数
+            const averageSurprisalDensity = bucket.TokenFragmentCount > 0
+                ? bucket.surprisalDensitySum / bucket.TokenFragmentCount
                 : 0;
-            const color = getSurprisalColor(averageSurprisal, 0.9);
+            const color = getByteSurprisalColor(averageSurprisalDensity, MINIMAP_COLOR_FACTOR);
             ctx.fillStyle = color;
 
             const y = (textAreaTop + bucket.y) / worldUnitsPerMinimapPixel; // 映射到minimap的y坐标
@@ -223,26 +224,6 @@ export class ScrollbarMinimap {
         });
     }
 
-
-    /**
-     * 计算token每个字符的惊讶度
-     * @param tokenIndex token索引
-     * @param renderData 渲染数据
-     * @returns 每个字符的惊讶度值
-     */
-    private getTokenSurprisalPerChar(tokenIndex: number, renderData: FrontendAnalyzeResult): number {
-        const token = renderData.bpe_strings?.[tokenIndex];
-        const realTopk = token?.real_topk;
-        if (Array.isArray(realTopk) && realTopk.length === 2) {
-            const [rank, prob] = realTopk;
-            const tokenSurprisal = -Math.log2(prob);
-            const tokenText = token?.raw || '';
-            const charCount = Array.from(tokenText).length;
-            return charCount > 0 ? tokenSurprisal / charCount : 0;
-        }
-        return 0;
-    }
-
     /**
      * 按Y坐标，把[y_min, y_max]范围内的token fragment聚合到bucketCount个桶
      * @param positions token 位置数组
@@ -271,7 +252,7 @@ export class ScrollbarMinimap {
         // 初始化桶数组，y坐标从y_min开始，依次递增bucketHeight
         const buckets: BucketData[] = Array.from({ length: bucketCount }, (_, bucketIndex) => ({
             y: y_min + bucketIndex * bucketHeight,
-            surprisalSum: 0,
+            surprisalDensitySum: 0,
             TokenFragmentCount: 0
         }));
 
@@ -288,10 +269,12 @@ export class ScrollbarMinimap {
 
             const bucket = buckets[bucketIndex];
 
-            // 计算该token每个字符的惊讶度并累加
-            const surprisalPerChar = this.getTokenSurprisalPerChar(pos.tokenIndex, renderData);
-            bucket.surprisalSum += surprisalPerChar;
+            // 计算该token字节平均惊讶度并累加
+            const token = renderData.bpe_strings[pos.tokenIndex];
+            const surprisalDensity = calculateSurprisalDensity(token);
+            bucket.surprisalDensitySum += surprisalDensity;
             bucket.TokenFragmentCount += 1;
+            // todo: 使用字节数加权计算bucket的平均信息密度，而不是按token平均计算
         });
 
         return {

client/src/ts/vis/SvgOverlayManager.ts

@@ -4,8 +4,8 @@
  */
 
 import {FrontendAnalyzeResult} from "../api/GLTR_API";
-import {calculateSurprisal, calculateSurprisalPerCharacter} from "../utils/Util";
-import {getSurprisalColor, getDiffColor} from "../utils/SurprisalColorConfig";
+import {calculateSurprisalDensity} from "../utils/Util";
+import {getByteSurprisalColor, getDiffColor} from "../utils/SurprisalColorConfig";
 import {TokenFragmentRect, RectCacheEntry} from "./types";
 import * as d3 from "d3";
 
@@ -16,8 +16,10 @@ export interface SvgOverlayManagerOptions {
     addTokenEventListeners: (element: SVGGElement, tokenIndex: number, rd: FrontendAnalyzeResult) => void;
     /** 差分模式：是否启用差分渲染 */
     diffMode?: boolean;
-    /** 差分数据：逐字的Δ(bit/char) */
-    deltaCharSurprisals?: number[];
+    /** 差分数据：逐字节的Δ信息密度(bits/Byte) */
+    deltaByteSurprisals?: number[];
+    /** 字符索引到字节索引的映射表 */
+    charToByteIndexMap?: number[];
 }
 
 export class SvgOverlayManager {
@@ -243,38 +245,46 @@ export class SvgOverlayManager {
         // 计算颜色
         let color: string;
         
-        if (this.options.diffMode && this.options.deltaCharSurprisals) {
-            // 差分模式：按字计算颜色
+        if (this.options.diffMode && this.options.deltaByteSurprisals) {
+            // 差分模式：按字节计算颜色
             const tokenData = rd.bpe_strings[pos.tokenIndex];
-            const offset = tokenData?.offset || [0, 0];
+            const offset = tokenData.offset;
             const charStart = offset[0];
             const charEnd = offset[1];
             
-            // 获取该token对应的字符范围内的差分值
-            const deltaCharSurprisals = this.options.deltaCharSurprisals;
-            const tokenCharDeltas: number[] = [];
+            // 获取该token对应的字节范围内的差分值
+            // token的offset是字符索引，需要通过映射表转换为字节索引
+            const deltaByteSurprisals = this.options.deltaByteSurprisals;
+            const charToByteIndexMap = this.options.charToByteIndexMap;
+            const tokenByteDeltas: number[] = [];
             
-            // 遍历token的字符范围，收集差分值
-            for (let i = charStart; i < charEnd && i < deltaCharSurprisals.length; i++) {
-                tokenCharDeltas.push(deltaCharSurprisals[i]);
+            // 必须有映射表才能正确转换字符索引到字节索引
+            if (!charToByteIndexMap || charToByteIndexMap.length === 0) {
+                color = getDiffColor(0);
+            } else {
+                // 将字符索引范围转换为字节索引范围
+                const byteStart = charToByteIndexMap[charStart] ?? charStart;
+                const byteEnd = charToByteIndexMap[charEnd] ?? charEnd;
+                
+                // 遍历token的字节范围，收集差分值
+                for (let byteIdx = byteStart; byteIdx < byteEnd && byteIdx < deltaByteSurprisals.length; byteIdx++) {
+                    tokenByteDeltas.push(deltaByteSurprisals[byteIdx]);
+                }
+                
+                // 计算平均差分值
+                let avgDelta = 0;
+                if (tokenByteDeltas.length > 0) {
+                    avgDelta = tokenByteDeltas.reduce((sum, val) => sum + val, 0) / tokenByteDeltas.length;
+                }
+                
+                // 使用统一的差分颜色配置
+                color = getDiffColor(avgDelta);
             }
-            
-            // 计算平均差分值
-            let avgDelta = 0;
-            if (tokenCharDeltas.length > 0) {
-                avgDelta = tokenCharDeltas.reduce((sum, val) => sum + val, 0) / tokenCharDeltas.length;
-            }
-            
-            // 使用统一的差分颜色配置
-            color = getDiffColor(avgDelta);
         } else {
             // 正常模式：使用surprisal计算颜色
             const tokenData = rd.bpe_strings[pos.tokenIndex];
-            const tokenText = tokenData?.raw || '';
-            const tokenTopK = this.options.getTokenRealTopk(rd, pos.tokenIndex);
-            const surprisal = tokenData ? calculateSurprisal(tokenTopK?.[1] || 0) : 0;
-            const surprisalPerChar = calculateSurprisalPerCharacter(surprisal, tokenText);
-            color = getSurprisalColor(surprisalPerChar);
+            const informationDensity = calculateSurprisalDensity(tokenData);
+            color = getByteSurprisalColor(informationDensity);
         }
 
         // 设置填充颜色

client/src/ts/vis/ToolTip.ts

@@ -1,4 +1,4 @@
-import { D3Sel, calculateSurprisal, calculateSurprisalPerCharacter } from "../utils/Util";
+import { D3Sel, calculateSurprisal, calculateSurprisalDensity } from "../utils/Util";
 import { SimpleEventHandler } from "../utils/SimpleEventHandler";
 import { GLTR_RenderItem } from "./GLTR_Text_Box";
 import * as d3 from "d3";
@@ -397,7 +397,7 @@ export class ToolTip {
      */
     private _doUpdate(ri: GLTR_RenderItem, event?: MouseEvent) {
         const { normalColor, selectedColor, detailColor, valueColor } = this.themeColors;
-        const predictions = Array.isArray(ri.others) ? ri.others : [];
+        const predictions = ri.tokenData.pred_topk;
         const hasPredictions = predictions.length > 0;
         const wScale = hasPredictions ? this._getScale(predictions[0][1]) : this._getScale(1);
 
@@ -406,12 +406,12 @@ export class ToolTip {
 
         // 更新当前token显示（第一行）
         this.currentToken.html(() => {
-            const visualizedToken = escapeHtml(visualizeSpecialChars(ri.token));
+            const visualizedToken = escapeHtml(visualizeSpecialChars(ri.tokenData.raw));
             return `<span style="color: ${selectedColor};">${visualizedToken}</span>`;
         });
 
         // 先设置内容，以便获取tooltip的实际尺寸
-        if (ri.bpeMerged) {
+        if (ri.tokenData.bpe_merged) {
             this.predictions.selectAll('.row').data([{ label: 'bpe_merged' }])
                 .join('div')
                 .attr('class', 'row info-row')
@@ -429,7 +429,7 @@ export class ToolTip {
                 .attr('class', 'row')
                 .style('display', 'table-row')
                 .html(d => {
-                    const color = ri.token != d[0] ? normalColor : selectedColor;
+                    const color = ri.tokenData.raw != d[0] ? normalColor : selectedColor;
                     const bar = '<div style="display: table-cell; width:110px;padding-left:5px;">' +
                         `<div style="display:inline-block;width: ${wScale(d[1])}px;background-color:${color};height: 10px;"></div>` +
                         ` <div style="display:inline-block;color: ${color};">${this.numF(d[1])}</div>` + "</div>";
@@ -442,13 +442,16 @@ export class ToolTip {
         }
 
             this.myDetail.html(() => {
-                const surprisal = calculateSurprisal(ri.prop);
-                const surprisalPerChar = calculateSurprisalPerCharacter(surprisal, ri.token);
+                const prob = ri.tokenData.real_topk[1];
+                const surprisal = calculateSurprisal(prob);
+                const informationDensity = calculateSurprisalDensity(ri.tokenData);
+                const utf8Size = new TextEncoder().encode(ri.tokenData.raw).length;
     
-                const surprisalPerCharText = `<span style="color: ${detailColor}">surprisalPerChar:</span> <span style="color: ${valueColor}">${this.significantF(surprisalPerChar)}</span> <span style="color: ${detailColor}">bits/char</span>`
+                const informationDensityText = `<span style="color: ${detailColor}">surprisal density:</span> <span style="color: ${valueColor}">${this.significantF(informationDensity)}</span> <span style="color: ${detailColor}">bits/Byte</span>`
+                const utf8SizeText = `<span style="color: ${detailColor}">utf8 size:</span> <span style="color: ${valueColor}">${utf8Size}</span> <span style="color: ${detailColor}">bytes</span>`
                 const surprisalText = `<span style="color: ${detailColor}">surprisal:</span> <span style="color: ${valueColor}">${this.significantF(surprisal)}</span> <span style="color: ${detailColor}">bits</span>`
-                const prop = `<span style="color: ${detailColor}">prob:</span> <span style="color: ${valueColor}">${this.significantF(ri.prop)}</span>`    
-                return `${surprisalPerCharText}<br/>${surprisalText}<br/>${prop}`
+                const prop = `<span style="color: ${detailColor}">prob:</span> <span style="color: ${valueColor}">${this.significantF(prob)}</span>`    
+                return `${informationDensityText}<br/>${utf8SizeText}<br/>${surprisalText}<br/>${prop}`
     
             })