增强语义分析功能：噪声分布拟合截尾对数正态分布，扫描-拟合迭代查找尾部信号阈值点，增加后验加权pw score染色；语义结果缓存

backend/api/analyze_semantic.py

@@ -25,23 +25,21 @@ def _log_request(query, text, client_ip=None):
 
 
 def _build_success_response(result, debug_info: bool = False):
-    """构建成功响应。debug_info=True 时包含 debug_abbrev、debug_top10"""
+    """构建成功响应。debug_info=True 时包含 debug_info 对象（abbrev、topk_tokens、topk_probs）"""
     resp = {
         "success": True,
         "model": result["model"],
         "token_attention": result["token_attention"],
         "full_match_degree": result["full_match_degree"],
     }
-    if debug_info:
-        if "debug_abbrev" in result:
-            resp["debug_abbrev"] = result["debug_abbrev"]
-        if "debug_top10" in result:
-            resp["debug_top10"] = result["debug_top10"]
+    if debug_info and "debug_info" in result:
+        resp["debug_info"] = result["debug_info"]
     return resp
 
 
 def _generate_semantic_events(
-    query: str, text: str, submode: Optional[str] = None, debug_info: bool = False, client_ip: Optional[str] = None
+    query: str, text: str, submode: Optional[str] = None, debug_info: bool = False,
+    full_match_degree_only: bool = False, client_ip: Optional[str] = None
 ):
     """
     流式语义分析核心：生成 SSE 事件流（progress + result/error）。
@@ -77,7 +75,7 @@ def _generate_semantic_events(
             try:
                 from backend.access_log import log_analyze_semantic_start
                 log_analyze_semantic_start(request_id, lock_wait_time, stream_mode=True)
-                result = _analyze_semantic(query, text, submode_override=submode, progress_callback=progress_callback, debug_info=debug_info)
+                result = _analyze_semantic(query, text, submode_override=submode, progress_callback=progress_callback, debug_info=debug_info, full_match_degree_only=full_match_degree_only)
                 analysis_result = result
             finally:
                 _inference_lock.release()
@@ -139,16 +137,18 @@ def _generate_semantic_events(
 
 
 def _analyze_semantic_with_stream(
-    query: str, text: str, submode: Optional[str] = None, debug_info: bool = False, client_ip: Optional[str] = None
+    query: str, text: str, submode: Optional[str] = None, debug_info: bool = False,
+    full_match_degree_only: bool = False, client_ip: Optional[str] = None
 ):
     """流式语义分析，通过 SSE 返回阶段级进度"""
     return SSEProgressReporter(
-        lambda: _generate_semantic_events(query, text, submode, debug_info, client_ip)
+        lambda: _generate_semantic_events(query, text, submode, debug_info, full_match_degree_only, client_ip)
     ).create_response()
 
 
 def _analyze_semantic_plain(
-    query: str, text: str, submode: Optional[str] = None, debug_info: bool = False, client_ip: Optional[str] = None
+    query: str, text: str, submode: Optional[str] = None, debug_info: bool = False,
+    full_match_degree_only: bool = False, client_ip: Optional[str] = None
 ):
     """
     非流式语义分析：封装流式实现，消费事件流后返回 JSON。
@@ -158,7 +158,7 @@ def _analyze_semantic_plain(
     error_msg = None
     status_code = 500
     try:
-        for event_str in _generate_semantic_events(query, text, submode, debug_info, client_ip):
+        for event_str in _generate_semantic_events(query, text, submode, debug_info, full_match_degree_only, client_ip):
             if not event_str.startswith('data: '):
                 continue
             data = json.loads(event_str[6:].strip())
@@ -199,6 +199,7 @@ def analyze_semantic(semantic_request):
     stream = semantic_request.get("stream", False)
     submode = (semantic_request.get("submode") or "").strip() or None
     debug_info = bool(semantic_request.get("debug_info", False))
+    full_match_degree_only = bool(semantic_request.get("full_match_degree_only", False))
 
     if not query:
         return {"success": False, "message": "缺少 query 字段"}, 400
@@ -207,5 +208,5 @@ def analyze_semantic(semantic_request):
 
     client_ip = get_client_ip()
     if stream:
-        return _analyze_semantic_with_stream(query, text, submode, debug_info, client_ip)
-    return _analyze_semantic_plain(query, text, submode, debug_info, client_ip)
+        return _analyze_semantic_with_stream(query, text, submode, debug_info, full_match_degree_only, client_ip)
+    return _analyze_semantic_plain(query, text, submode, debug_info, full_match_degree_only, client_ip)

backend/semantic_analyzer.py

@@ -12,6 +12,7 @@ count/fill_blank 按概率加权（Σ pᵢ·zᵢ）。
 """
 
 import gc
+import math
 from typing import Callable, Dict, List, Optional
 
 import torch
@@ -58,6 +59,7 @@ def _analyze_logits_gradient(
     submode_override: Optional[str] = None,
     progress_callback: Optional[Callable[[int, int, str, Optional[int]], None]] = None,
     debug_info: bool = False,
+    full_match_degree_only: bool = False,
 ) -> Dict:
     """
     梯度归因：logits 对输入 embedding 的梯度。
@@ -147,13 +149,12 @@ def _analyze_logits_gradient(
                 attention_mask=attention_mask,
                 output_attentions=False,
             )
-        # 显式同步，确保前向已完成，progress_callback 时机准确
+        # 显式同步，确保已完成，progress_callback 时机准确
         if device.type == "cuda":
             torch.cuda.synchronize(device)
         elif device.type == "mps":
             torch.mps.synchronize()
-        if progress_callback:
-            progress_callback(3, TOTAL_STEPS, "backward", None)
+
         logits = outputs.logits[:, -1, :]
         topk_vals, topk_ids = torch.topk(logits, LOGITS_GRADIENT_TOPK, dim=-1)
         probs = torch.softmax(logits, dim=-1)
@@ -163,6 +164,19 @@ def _analyze_logits_gradient(
 
         neg_token = "无" if submode == "fill_blank" else "0"
         neg_id = tokenizer.encode(neg_token, add_special_tokens=False)[0]
+        # 全文匹配度：count/match_score 用 1-P("0")，fill_blank 用 1-P("无")
+        p_neg = probs[0, neg_id].item()
+        full_match_degree = round(1.0 - p_neg, 4)
+
+        if full_match_degree_only and submode == "count":
+            return {
+                "model": get_semantic_model_display_name(),
+                "token_attention": [],
+                "full_match_degree": full_match_degree,
+            }
+
+        if progress_callback:
+            progress_callback(3, TOTAL_STEPS, "backward", None)
         # 归因目标：raw logits（不经过 softmax backward），避免饱和与竞争污染。
         if submode == "count" or submode == "fill_blank":
             # count/fill_blank 均用 top-10、按概率加权 Σ pᵢ·zᵢ，并排除 neg_token（0/无）以保持梯度方向与「相关」一致。
@@ -180,39 +194,43 @@ def _analyze_logits_gradient(
         else:
             raise ValueError(f"未知 submode: {submode}")
         target_logit.backward()
-
         grad = embeds.grad
         if grad is None:
             raise RuntimeError("logits_gradient: 梯度未回传，可能模型不支持（如 int8 量化）")
 
+        # 显式同步，确保已完成，progress_callback 时机准确
+        if device.type == "cuda":
+            torch.cuda.synchronize(device)
+        elif device.type == "mps":
+            torch.mps.synchronize()
         if progress_callback:
             progress_callback(4, TOTAL_STEPS, "processing", None)
+            
         text_token_end = len(offset_mapping)
         # 在 GPU 上一次性计算所有 token 的 ‖∇f‖，避免循环内 .item() 导致 500 次 GPU→CPU 同步
         grad_slice = grad[0, prompt_end:text_token_end].float()
         norms = grad_slice.norm(dim=-1).cpu().tolist()
         token_attention: List[Dict] = []
+        nan_count = 0
         for i in range(prompt_end, text_token_end):
             s, e = offset_mapping[i]
             if s >= text_start_char and e <= text_end_char:
                 s_rel, e_rel = s - text_start_char, e - text_start_char
-                token_attention.append({"offset": [s_rel, e_rel], "raw": truncated_text[s_rel:e_rel], "score": norms[i - prompt_end]})
-
-        # 全文匹配度：count/match_score 用 1-P("0")，fill_blank 用 1-P("无")
-        p_neg = probs[0, neg_id].item()
-        full_match_degree = round(1.0 - p_neg, 4)
+                score = norms[i - prompt_end]
+                if not math.isfinite(score):
+                    score = 0.0
+                    nan_count += 1
+                token_attention.append({"offset": [s_rel, e_rel], "raw": truncated_text[s_rel:e_rel], "score": score})
+        if nan_count > 0:
+            print(f"⚠️ token_attention 中有 {nan_count} 个 score 为 NaN/Inf，已替换为 0。")
 
-        model_display = get_semantic_model_display_name()
         out = {
-            "model": model_display,
+            "model": get_semantic_model_display_name(),
             "token_attention": token_attention,
-            "topk_tokens": topk_tokens,
-            "topk_probs": topk_probs,
             "full_match_degree": full_match_degree,
         }
         if debug_info:
-            out["debug_abbrev"] = abbrev
-            out["debug_top10"] = [{"token": t, "prob": p} for t, p in zip(topk_tokens, topk_probs)]
+            out["debug_info"] = {"abbrev": abbrev, "topk_tokens": topk_tokens, "topk_probs": topk_probs}
         return out
     finally:
         if use_gc:
@@ -227,6 +245,7 @@ def analyze_semantic(
     submode_override: Optional[str] = None,
     progress_callback: Optional[Callable[[int, int, str, Optional[int]], None]] = None,
     debug_info: bool = False,
+    full_match_degree_only: bool = False,
 ) -> Dict:
     """
     分析原文各 token 与 query 的相关度（使用 logits_gradient 梯度归因）。
@@ -236,10 +255,10 @@ def analyze_semantic(
         text: 原文
         submode_override: 评估时可选覆盖子模式（count/match_score/fill_blank）
         progress_callback: 可选进度回调 (step, total_steps, stage, percentage)
-        debug_info: 为 True 时返回 debug_abbrev（推理原文缩写）和 debug_top10
+        debug_info: 为 True 时返回 debug_abbrev（推理原文缩写）；topk_tokens、topk_probs 始终在结果中
 
     Returns:
-        {"model", "token_attention", "topk_tokens", "topk_probs"}
+        {"model", "token_attention", "full_match_degree"}；debug_info=True 时包含 debug_info 对象
     """
     tokenizer, model, device = ensure_semantic_loaded()
     return _analyze_logits_gradient(
@@ -247,4 +266,5 @@ def analyze_semantic(
         submode_override=submode_override,
         progress_callback=progress_callback,
         debug_info=debug_info,
+        full_match_degree_only=full_match_degree_only,
     )

client/src/css/_responsive.scss

@@ -246,9 +246,10 @@
   }
   
   // 调整浮动内容宽度，不使用自己的滚动条
+  // 使用 overflow: visible 避免下拉框（如查询历史）被裁剪；overflow-x: hidden 会令 overflow-y 被计算为 auto 从而产生裁剪
   .floating_content {
     @include full-width-adaptive;
-    @include mobile-scroll-container;
+    overflow: visible;
   }
   
   // 调整统计图容器

client/src/css/_semantic-analysis.scss

@@ -48,6 +48,16 @@
       align-items: center;
       gap: 6px;
 
+      .semantic-submode-group {
+        display: flex;
+        align-items: center;
+        gap: 6px;
+
+        &.semantic-submode-group-right {
+          margin-left: auto;
+        }
+      }
+
       .semantic-submode-label {
         font-size: 9pt;
         color: var(--text-muted);
@@ -199,9 +209,13 @@
     font-family: ui-monospace, "Cascadia Code", "Source Code Pro", Menlo, Consolas, "DejaVu Sans Mono", monospace;
     word-break: break-word;
     overflow-wrap: break-word;
-    // 颜色加重
     color: var(--text-color, #333);
   }
+
+  // TopK 图表：与 tooltip 一致，宽度更大，与上方打印区留间隔
+  .semantic-debug-topk-chart {
+    margin-top: 20px;
+  }
 }
 
 // 语义分析阶段级进度（与 analyze-progress 同风格，绝对定位不占布局空间）

client/src/css/start.scss

@@ -565,7 +565,6 @@ select {
   // 白天模式使用默认字重（400），夜间模式使用 Light 字重（300）
   background-color: var(--text-area-bg);  // 使用CSS变量控制背景色
   color: var(--text-color);  // 使用CSS变量控制文字颜色
-  transition: background-color 0.3s ease, color 0.3s ease;  // 平滑过渡
   // 确保至少350px高度以容纳tooltip（内容不足时生效）
   min-height: 350px;
   // 不设置固定padding-bottom，让内容自然决定高度

client/src/index.html

@@ -150,13 +150,23 @@
                             </div>
                         </div>
                         <div class="semantic-submode-row">
-                            <label class="semantic-submode-label" for="semantic_submode_select">submode: </label>
-                            <select id="semantic_submode_select" class="semantic-submode-select">
-                                <option value="count">count</option>
-                                <option value="match_score">match_score</option>
-                                <option value="fill_blank">fill_blank</option>
-                                <option value="hybrid">hybrid</option>
-                            </select>
+                            <span class="semantic-submode-group">
+                                <label class="semantic-submode-label" for="semantic_submode_select">submode: </label>
+                                <select id="semantic_submode_select" class="semantic-submode-select">
+                                    <option value="count">count</option>
+                                    <option value="match_score">match_score</option>
+                                    <option value="fill_blank">fill_blank</option>
+                                    <option value="hybrid">hybrid</option>
+                                </select>
+                            </span>
+                            <span class="semantic-submode-group semantic-submode-group-right">
+                                <label class="semantic-submode-label" for="semantic_color_source_select">color source: </label>
+                                <select id="semantic_color_source_select" class="semantic-submode-select">
+                                    <option value="raw_score_normed" selected>raw score normed</option>
+                                    <option value="signal_probability">signal probability</option>
+                                    <option value="pw_score">pw score</option>
+                                </select>
+                            </span>
                         </div>
                     </div>
                 </section>
@@ -164,9 +174,9 @@
 
             <section id="all_result" class="results-section">
                 <div id="stats" class="stats-container">
-                    <div id="semantic_score_histogram_item" class="histogram-item" style="display: none;">
-                        <div id="semantic_score_histogram_title"></div>
-                        <svg id="stats_semantic_score"></svg>
+                    <div id="raw_score_normed_histogram_item" class="histogram-item" style="display: none;">
+                        <div id="raw_score_normed_histogram_title"></div>
+                        <svg id="stats_raw_score_normed"></svg>
                     </div>
                     <div id="token_histogram_item" class="histogram-item" style="display: none;">
                         <div id="token_histogram_title"></div>

client/src/package.json

@@ -5,6 +5,10 @@
   "main": "webpack.config.js",
   "scripts": {
     "test": "echo \"Error: no test specified\" && exit 1",
+    "test:lognormal": "npx tsx ts/utils/visualizationUpdater.lognormal.test.ts",
+    "test:lognormal:tau": "npx tsx ts/utils/lognormalFit.tauBoundary.test.ts",
+    "test:signalThreshold": "npx tsx ts/utils/signalThresholdDetector.1log.test.ts",
+    "demo:histogramCdf": "npx tsx ts/utils/histogramCdfDemoData.ts",
     "prebuild": "node scripts/updateIntroHTML.js",
     "prebuild:dev": "node scripts/updateIntroHTML.js",
     "wp": "npm run build:dev",

client/src/ts/api/GLTR_API.ts

@@ -5,6 +5,7 @@ Attn API and Types
 import * as d3 from "d3";
 import URLHandler from "../utils/URLHandler";
 import {cleanSpecials} from "../utils/Util";
+import * as semanticResultCache from "../utils/semanticResultCache";
 import {AnalyzeResponse, AnalyzeResult, TokenWithOffset} from "./generatedSchemas";
 
 export type FrontendToken = TokenWithOffset & { bpe_merged?: boolean };
@@ -254,28 +255,38 @@ export class TextAnalysisAPI {
         query: string,
         text: string,
         onProgress?: (step: number, totalSteps: number, stage: string, percentage?: number) => void,
-        submode?: string
-    ): Promise<{ success: boolean; model?: string; token_attention?: Array<{ offset: [number, number]; raw: string; score: number }>; topk_tokens?: string[]; topk_probs?: number[]; full_match_degree?: number; message?: string }> {
+        submode?: string,
+        fullMatchDegreeOnly?: boolean
+    ): Promise<{ success: boolean; model?: string; token_attention?: Array<{ offset: [number, number]; raw: string; score: number }>; debug_info?: { abbrev?: string; topk_tokens?: string[]; topk_probs?: number[] }; full_match_degree?: number; message?: string }> {
         if (submode === 'hybrid') {
-            const r1 = await this.analyzeSemantic(query, text, onProgress, 'count');
+            const r1 = await this.analyzeSemantic(query, text, onProgress, 'count', true);
             const r2 = await this.analyzeSemantic(query, text, onProgress, 'fill_blank');
-            return { ...r2, full_match_degree: r1.full_match_degree };
+            const fromCache = (r1 as { __fromCache?: boolean }).__fromCache && (r2 as { __fromCache?: boolean }).__fromCache;
+            return { ...r2, full_match_degree: r1.full_match_degree, __fromCache: fromCache } as typeof r2 & { __fromCache?: boolean };
         }
-        if (onProgress) {
-            return this.analyzeSemanticWithProgress(query, text, onProgress, submode);
-        }
-        const payload: Record<string, unknown> = { query, text, debug_info: true };
-        if (submode) payload.submode = submode;
-        return d3.json(this.baseURL + '/api/analyze-semantic', {
-            method: 'POST',
-            body: JSON.stringify(payload),
-            headers: this.getHeaders()
-        }).then((response: any) => {
+        const cacheSubmode = submode;
+        const cached = semanticResultCache.get(text, query, cacheSubmode);
+        if (cached && (fullMatchDegreeOnly || cached.token_attention)) return { ...cached, __fromCache: true } as typeof cached & { __fromCache?: boolean };
+        const doRequest = async (): Promise<typeof cached> => {
+            if (onProgress) {
+                return this.analyzeSemanticWithProgress(query, text, onProgress, submode, fullMatchDegreeOnly);
+            }
+            const payload: Record<string, unknown> = { query, text, debug_info: true };
+            if (submode) payload.submode = submode;
+            if (fullMatchDegreeOnly) payload.full_match_degree_only = true;
+            const response = await d3.json(this.baseURL + '/api/analyze-semantic', {
+                method: 'POST',
+                body: JSON.stringify(payload),
+                headers: this.getHeaders()
+            }) as any;
             if (response && response.success === false) {
                 throw new Error(response.message || 'Semantic analysis failed');
             }
             return response;
-        });
+        };
+        const res = await doRequest();
+        if (res?.success) semanticResultCache.set(text, query, res, cacheSubmode);
+        return res;
     }
 
     /**
@@ -285,10 +296,12 @@ export class TextAnalysisAPI {
         query: string,
         text: string,
         onProgress: (step: number, totalSteps: number, stage: string, percentage?: number) => void,
-        submode?: string
-    ): Promise<{ success: boolean; model?: string; token_attention?: Array<{ offset: [number, number]; raw: string; score: number }>; topk_tokens?: string[]; topk_probs?: number[]; full_match_degree?: number; message?: string }> {
+        submode?: string,
+        fullMatchDegreeOnly?: boolean
+    ): Promise<{ success: boolean; model?: string; token_attention?: Array<{ offset: [number, number]; raw: string; score: number }>; debug_info?: { abbrev?: string; topk_tokens?: string[]; topk_probs?: number[] }; full_match_degree?: number; message?: string }> {
         const payload: Record<string, unknown> = { query, text, stream: true, debug_info: true };
         if (submode) payload.submode = submode;
+        if (fullMatchDegreeOnly) payload.full_match_degree_only = true;
         return this.fetchSSEStream(
             '/api/analyze-semantic',
             payload,
@@ -386,7 +399,10 @@ export class TextAnalysisAPI {
                 reject(new Error(parsed.message || errorMessage));
             }
         } catch (e) {
-            console.warn('Failed to parse SSE message:', e, data);
+            const msg = e instanceof SyntaxError
+                ? `SSE 数据解析失败：${e.message}（可能是后端返回了无效 JSON，如 NaN）`
+                : `SSE 消息处理失败：${e instanceof Error ? e.message : String(e)}`;
+            reject(new Error(msg));
         }
     }
 

client/src/ts/appInitializer.ts

@@ -6,7 +6,7 @@
 import * as d3 from 'd3';
 import { SimpleEventHandler } from './utils/SimpleEventHandler';
 import { TextAnalysisAPI } from './api/GLTR_API';
-import { getTokenSurprisalColor, getByteSurprisalColor } from './utils/SurprisalColorConfig';
+import { getTokenSurprisalColor, getByteSurprisalColor, HISTOGRAM_MIN_ALPHA } from './utils/SurprisalColorConfig';
 
 /**
  * 公共初始化返回对象
@@ -33,8 +33,8 @@ export function initializeCommonApp(apiPrefix: string = '', element?: Element):
     return {
         eventHandler: new SimpleEventHandler(targetElement),
         api: new TextAnalysisAPI(apiPrefix),
-        tokenSurprisalColorScale: getTokenSurprisalColor,
-        byteSurprisalColorScale: getByteSurprisalColor,
+        tokenSurprisalColorScale: (v) => getTokenSurprisalColor(v, HISTOGRAM_MIN_ALPHA),
+        byteSurprisalColorScale: (v) => getByteSurprisalColor(v, 1, HISTOGRAM_MIN_ALPHA),
         totalSurprisalFormat: (n: number | null) => n !== null && Number.isFinite(n) ? format(n) : String(n)
     };
 }

client/src/ts/compare.ts

@@ -348,6 +348,7 @@ window.onload = () => {
             colorScale: tokenSurprisalColorScale,
             averageValue: stats.tokenAverage ?? undefined,
             p90Value: stats.tokenP90 ?? undefined,
+            p90Label: tokenHistogramConfig.averageLabel,
         });
 
         // 更新列视图中 token surprisal histogram 的标题文本
@@ -1270,11 +1271,11 @@ window.onload = () => {
     // 初始化主题管理器（在所有函数定义之后）
     const themeManager = initThemeManager({
         onThemeChange: () => {
-            // 主题切换时重新渲染所有图表
             columnsData.forEach((col) => {
                 if (col.data && col.stats) {
                     renderStatsForColumn(col.id, col);
                 }
+                requestAnimationFrame(() => col.lmfInstance?.reRenderCurrent());
             });
         }
     });

client/src/ts/controllers/highlightController.ts

@@ -3,14 +3,15 @@ import type { GLTR_Text_Box } from '../vis/GLTR_Text_Box';
 import type { Histogram } from '../vis/Histogram';
 import type { HistogramBinClickEvent } from '../vis/Histogram';
 import type { FrontendAnalyzeResult } from '../api/GLTR_API';
-import { calculateHighlights, type HistogramType } from '../utils/highlightUtils';
+import { calculateHighlights, type HistogramType, type HighlightData } from '../utils/highlightUtils';
+
+export type HighlightCurrentData = { result: FrontendAnalyzeResult; signalProbs?: number[]; pPwValues?: number[]; pwScores?: number[] } | null;
 
 export type HighlightControllerOptions = {
     stats_frac: Histogram;
-    /** Semantic score histogram（可选） */
-    stats_semantic_score?: Histogram;
+    stats_raw_score_normed?: Histogram;
     lmf: GLTR_Text_Box;
-    currentData: { result: FrontendAnalyzeResult } | null;
+    currentData: HighlightCurrentData;
 };
 
 export class HighlightController {
@@ -25,7 +26,7 @@ export class HighlightController {
      */
     public clearHighlights(): void {
         this.options.stats_frac.clearSelection();
-        this.options.stats_semantic_score?.clearSelection();
+        this.options.stats_raw_score_normed?.clearSelection();
         this.options.lmf.clearHighlight();
     }
 
@@ -43,32 +44,32 @@ export class HighlightController {
         }
 
         const { x0, x1, binIndex, no_bins, source } = ev;
-        const data = currentData.result;
+        const highlightData: HighlightData = { ...currentData.result, signalProbs: currentData.signalProbs, pPwValues: currentData.pPwValues, pwScores: currentData.pwScores };
 
-        // 首页：根据直方图 source 区分类型
         let histogramType: HistogramType = 'token';
-        if (source === 'stats_semantic_score') {
-            histogramType = 'semantic';
-        }
+        if (source === 'stats_raw_score_normed') histogramType = 'raw_score_normed';
 
-        // 同一视图内仅保持一个直方图选中状态
-        if (histogramType === 'semantic') {
+        if (histogramType === 'raw_score_normed') {
             this.options.stats_frac.clearSelection();
         } else {
-            this.options.stats_semantic_score?.clearSelection();
+            this.options.stats_raw_score_normed?.clearSelection();
         }
 
-        const { indices, style } = calculateHighlights(histogramType, x0, x1, binIndex, no_bins, data);
+        const { indices, style } = calculateHighlights(histogramType, x0, x1, binIndex, no_bins, highlightData);
 
         this.options.lmf.setHighlightedIndices(indices, style);
     }
 
+    /** 获取当前高亮数据 */
+    public getCurrentData(): HighlightCurrentData {
+        return (this.options as { currentData: HighlightCurrentData }).currentData;
+    }
+
     /**
      * 更新当前数据（当数据变化时调用）
      */
-    public updateCurrentData(currentData: { result: FrontendAnalyzeResult } | null): void {
-        // 创建一个新对象来更新，保持 options 对象的引用不变
-        (this.options as any).currentData = currentData;
+    public updateCurrentData(currentData: HighlightCurrentData): void {
+        (this.options as { currentData: HighlightCurrentData }).currentData = currentData;
     }
 }
 

client/src/ts/lang/translations.ts

@@ -188,12 +188,18 @@ export const translations: Translations = {
         'information per token histogram': 'token信息量直方图',
         'information per token progress': 'token信息量进度图',
         'token index': 'token索引',
-        'semantic score histogram': '语义得分直方图',
-        'score': '得分',
+        'raw score normed histogram': '归一化原始分数直方图',
+        'semantic signal prob histogram': '语义信号概率直方图',
+        'signal prob': 'signal概率',
+        'signal ratio': '信号比',
+        'pw score': 'pw 分数',
 
         // ========== Tooltip 内容 ==========
         'information density:': '信息密度：',
-        'match score:': '匹配度：',
+        'pw score:': 'pw 分数：',
+        'signal prob:': 'signal概率：',
+        'signal probability:': '信号概率：',
+        'raw score normed:': '归一化原始分数：',
         'Match: {0}%': '匹配度: {0}%',
         'raw score:': '原始分数：',
         'prob:': '概率：',

client/src/ts/start.ts

@@ -47,6 +47,7 @@ import { isValidUrl, extractUrl, isPureUrl } from './utils/urlUtils';
 import { AdminManager } from './utils/adminManager';
 import { SettingsMenuManager } from './utils/settingsMenuManager';
 import { saveHistory, initQueryHistoryDropdown } from './utils/queryHistory';
+import { removeByQuery as removeSemanticCacheByQuery } from './utils/semanticResultCache';
 import { playAnalysisCompleteSound } from './utils/soundNotification';
 
 const current = {
@@ -170,7 +171,7 @@ window.onload = () => {
         width: 400,   // 宽度
         height: 200   // 增加高度从默认150px到200px
     });
-    const stats_semantic_score = new Histogram(d3.select('#stats_semantic_score'), eventHandler, {
+    const stats_raw_score_normed = new Histogram(d3.select('#stats_raw_score_normed'), eventHandler, {
         width: 400,
         height: 200
     });
@@ -202,7 +203,7 @@ window.onload = () => {
     // 创建高亮控制器
     const highlightController = new HighlightController({
         stats_frac,
-        stats_semantic_score,
+        stats_raw_score_normed,
         lmf,
         currentData: null
     });
@@ -217,7 +218,7 @@ window.onload = () => {
         highlightController,
         textInputController,
         stats_frac,
-        stats_semantic_score,
+        stats_raw_score_normed,
         stats_surprisal_progress,
         appStateManager,
         surprisalColorScale: tokenSurprisalColorScale as d3.ScaleSequential<string>
@@ -225,7 +226,9 @@ window.onload = () => {
 
     // 初始化主题管理器（在设置菜单中）
     const themeManager = initThemeManager({
-        onThemeChange: () => visualizationUpdater.rerenderHistograms()
+        onThemeChange: () => {
+            visualizationUpdater.rerenderOnThemeChange();
+        }
     }, '#theme_dropdown');
 
     // 初始化语言管理器（在设置菜单中）
@@ -276,6 +279,47 @@ window.onload = () => {
     // Semantic analysis UI 完全由配置决定，初始化时同步
     visualizationUpdater.syncSemanticUiFromConfig();
 
+    // 语义分析：从 URL 参数恢复查询输入和选项（刷新后可恢复）
+    const initSemanticFromUrl = () => {
+        const params = URLHandler.parameters;
+        const query = params['semantic_query'];
+        const submode = params['semantic_submode'];
+        const colorSource = params['semantic_color_source'];
+        const validSubmodes = ['count', 'match_score', 'fill_blank', 'hybrid'];
+        const validColorSources = ['raw_score_normed', 'signal_probability', 'pw_score'];
+        if (typeof query === 'string') {
+            const el = document.getElementById('semantic_search_input') as HTMLInputElement | null;
+            if (el) el.value = query;
+        }
+        if (typeof submode === 'string' && validSubmodes.includes(submode)) {
+            const el = document.getElementById('semantic_submode_select') as HTMLSelectElement | null;
+            if (el) el.value = submode;
+        }
+        if (typeof colorSource === 'string' && validColorSources.includes(colorSource)) {
+            const el = document.getElementById('semantic_color_source_select') as HTMLSelectElement | null;
+            if (el) el.value = colorSource;
+        }
+    };
+    initSemanticFromUrl();
+
+    // 语义分析：同步查询和选项到 URL（刷新后可恢复）
+    const syncSemanticToUrl = () => {
+        const queryEl = document.getElementById('semantic_search_input') as HTMLInputElement | null;
+        const submodeEl = document.getElementById('semantic_submode_select') as HTMLSelectElement | null;
+        const colorEl = document.getElementById('semantic_color_source_select') as HTMLSelectElement | null;
+        const query = queryEl?.value?.trim() ?? '';
+        const submode = submodeEl?.value?.trim() ?? '';
+        const colorSource = colorEl?.value?.trim() ?? '';
+        const currentParams = URLHandler.parameters;
+        if (query) currentParams['semantic_query'] = query;
+        else delete currentParams['semantic_query'];
+        if (submode) currentParams['semantic_submode'] = submode;
+        else delete currentParams['semantic_submode'];
+        if (colorSource) currentParams['semantic_color_source'] = colorSource;
+        else delete currentParams['semantic_color_source'];
+        URLHandler.updateUrl(currentParams, false);
+    };
+
     // *****************************
     // *****  demo stuff *****
     // *****************************
@@ -805,7 +849,8 @@ window.onload = () => {
                 visualizationUpdater.handleSemanticResponse(res, text);
                 appStateManager.setLastSearchedQuery(query);
                 saveHistory(query);
-                playAnalysisCompleteSound();
+                syncSemanticToUrl();
+                if (!(res as { __fromCache?: boolean }).__fromCache) playAnalysisCompleteSound();
                 const md = res?.full_match_degree;
                 const mdEl = d3.select('#semantic_match_degree');
                 if (md != null && typeof md === 'number') {
@@ -838,7 +883,14 @@ window.onload = () => {
         input: semanticSearchInput,
         dropdownId: 'semantic_search_history_dropdown',
         onSelect: () => appStateManager.updateButtonStates(),
-        onHistorySelect: runSemanticSearch
+        onHistorySelect: runSemanticSearch,
+        onRemove: removeSemanticCacheByQuery
+    });
+    semanticSearchInput?.addEventListener('blur', syncSemanticToUrl);
+    document.getElementById('semantic_submode_select')?.addEventListener('change', syncSemanticToUrl);
+    document.getElementById('semantic_color_source_select')?.addEventListener('change', () => {
+        visualizationUpdater.updateSemanticColorSource();
+        syncSemanticToUrl();
     });
 
     // Save按钮点击事件（使用 serverDemoController）

client/src/ts/utils/SurprisalColorConfig.ts

@@ -26,15 +26,18 @@ export const MINIMAP_COLOR_FACTOR = 1.3;
 const SURPRISAL_RED_RGB = "255, 71, 64";
 const SURPRISAL_MAX_ALPHA = 0.7;
 
+/** 直方图渐变最浅色 alpha 下限（10% 区间），供直方图使用方配置 */
+export const HISTOGRAM_MIN_ALPHA = 0.1 * SURPRISAL_MAX_ALPHA;
+
 /**
  * 根据归一化值获取对应的颜色（输入值应在[0,1]区间）
  * @param normalizedValue 归一化后的值，范围[0,1]
- * @returns 颜色字符串（rgba格式，从透明到红色）
+ * @param minAlpha alpha 下限，默认不限制
  */
-
-export function getSurprisalColorNormalized(normalizedValue: number): string {
+export function getSurprisalColorNormalized(normalizedValue: number, minAlpha?: number): string {
     const clampedValue = Math.max(0, Math.min(1, normalizedValue));
-    const alpha = clampedValue * SURPRISAL_MAX_ALPHA;
+    let alpha = clampedValue * SURPRISAL_MAX_ALPHA;
+    if (minAlpha != null) alpha = Math.max(minAlpha, alpha);
     return `rgba(${SURPRISAL_RED_RGB}, ${alpha})`;
 }
 
@@ -57,32 +60,33 @@ function normalizeTo_01(value: number, maxValue: number): number {
 /**
  * 根据token惊讶度值获取对应的颜色（线性映射，不取整）
  * @param surprisal token惊讶度值，范围[0, TOKEN_SURPRISAL_MAX]会被映射到[0, 1]，超出范围会被截断
- * @returns 颜色字符串（rgba格式）
+ * @param minAlpha alpha 下限，默认不限制
  */
-export function getTokenSurprisalColor(surprisal: number): string {
+export function getTokenSurprisalColor(surprisal: number, minAlpha?: number): string {
     const normalizedValue = normalizeTo_01(surprisal, TOKEN_SURPRISAL_MAX);
-    return getSurprisalColorNormalized(normalizedValue);
+    return getSurprisalColorNormalized(normalizedValue, minAlpha);
 }
 
 /**
  * 根据byte密度惊讶度值获取对应的颜色（线性映射，不取整）
  * @param byteSurprisal byte密度惊讶度值，范围[0, BYTE_SURPRISAL_MAX]会被映射到[0, 1]，超出范围会被截断
  * @param colorFactor 颜色因子，用于调整颜色强度，目前主要为了minimap显示更明显（平均后byte surprisal密度会过小，所以需要放大）。默认为1
- * @returns 颜色字符串（rgba格式）
+ * @param minAlpha alpha 下限，默认不限制
  */
-export function getByteSurprisalColor(byteSurprisal: number, colorFactor: number = 1): string {
+export function getByteSurprisalColor(byteSurprisal: number, colorFactor: number = 1, minAlpha?: number): string {
     const normalizedValue = normalizeTo_01(byteSurprisal * colorFactor, BYTE_SURPRISAL_MAX);
-    return getSurprisalColorNormalized(normalizedValue);
+    return getSurprisalColorNormalized(normalizedValue, minAlpha);
 }
 
 /**
- * 根据 matchScore 获取颜色（用于语义匹配度染色）
- * @param matchScore 归一化分数，范围 [0, 1]
+ * 根据 rawScoreNormed 获取颜色（用于语义匹配度染色）
+ * @param rawScoreNormed 归一化分数，范围 [0, 1]
+ * @param minAlpha alpha 下限，默认不限制
  */
-export function getSemanticSimilarityColor(matchScore: number): string {
-    if (!isFiniteNumber(matchScore)) return 'transparent';
-    const normalizedValue = normalizeTo_01(matchScore, SEMANTIC_SIMILARITY_MAX);
-    return getSurprisalColorNormalized(normalizedValue);
+export function getSemanticSimilarityColor(rawScoreNormed: number, minAlpha?: number): string {
+    if (!isFiniteNumber(rawScoreNormed)) return 'transparent';
+    const normalizedValue = normalizeTo_01(rawScoreNormed, SEMANTIC_SIMILARITY_MAX);
+    return getSurprisalColorNormalized(normalizedValue, minAlpha);
 }
 
 // ==========================================

client/src/ts/utils/fitQuality.ts

@@ -0,0 +1,39 @@
+/**
+ * 拟合质量计算（纯数学，无 Node 依赖）
+ */
+
+import { logNormalCdf } from './lognormalFit';
+
+/**
+ * 计算截尾对数正态在拟合区间内的拟合质量（仅用拟合数据）
+ * @returns { maxDiff, rmse, maxDiffIdx } maxDiff = max|CDF_trunc - ECDF|，rmse = sqrt(mean(diff²))
+ */
+export function computeFitQuality(
+    noise: number[],
+    tau: number,
+    mu: number,
+    sigma: number
+): { maxDiff: number; rmse: number; maxDiffIdx: number } {
+    const nNoise = noise.length;
+    if (nNoise < 1) return { maxDiff: NaN, rmse: NaN, maxDiffIdx: -1 };
+    const F_tau = logNormalCdf(tau, mu, sigma);
+    const cdfTrunc = (x: number) =>
+        x <= 0 ? 0 : x >= tau ? 1 : logNormalCdf(x, mu, sigma) / F_tau;
+
+    let maxDiff = 0;
+    let maxDiffIdx = 0;
+    let sumSqDiff = 0;
+    for (let i = 0; i < nNoise; i++) {
+        const x = noise[i]!;
+        const ecdf = (i + 1) / nNoise;
+        const cdf = cdfTrunc(x);
+        const diff = cdf - ecdf;
+        if (Math.abs(diff) > maxDiff) {
+            maxDiff = Math.abs(diff);
+            maxDiffIdx = i;
+        }
+        sumSqDiff += diff * diff;
+    }
+    const rmse = Math.sqrt(sumSqDiff / nNoise);
+    return { maxDiff, rmse, maxDiffIdx };
+}

client/src/ts/utils/highlightUtils.ts

@@ -2,13 +2,25 @@ import type { FrontendAnalyzeResult } from '../api/GLTR_API';
 import { calculateSurprisal, calculateSurprisalDensity } from './Util';
 import { extractRealTopkFromTokens } from './tokenUtils';
 
+/** 首/末 bin 包含超出范围的值，中间 bin 为 [x0, x1) */
+function valueInBinRange(value: number, x0: number, x1: number, binIndex: number, no_bins: number): boolean {
+    const isFirstBin = binIndex === 0;
+    const isLastBin = binIndex === no_bins - 1;
+    if (isFirstBin) return value < x1;
+    if (isLastBin) return value >= x0;
+    return value >= x0 && value < x1;
+}
+
+export type HistogramType = 'token' | 'byte' | 'raw_score_normed';
+export type HighlightData = FrontendAnalyzeResult & { rawScoresNormed?: number[]; attentionRawScores?: number[]; signalProbs?: number[]; pPwValues?: number[]; pwScores?: number[] };
+
 /**
  * 根据直方图 bin 的范围计算需要高亮的 token 索引集合（基于 token surprisal）
  * @param x0 bin 起始值
  * @param x1 bin 结束值
  * @param binIndex bin在bins数组中的索引
  * @param no_bins 直方图的总bin数量
- * @param result 前端分析结果（包含 originalTokens、mergedTokens、originalToMergedMap）
+ * @param data 前端分析结果（包含 originalTokens、mergedTokens、originalToMergedMap）
  * @returns 需要高亮的 merged token 索引集合
  */
 export function calculateTokenSurprisalHighlights(
@@ -16,43 +28,23 @@ export function calculateTokenSurprisalHighlights(
     x1: number,
     binIndex: number,
     no_bins: number,
-    result: FrontendAnalyzeResult
+    data: HighlightData
 ): Set<number> {
     const highlightedIndices = new Set<number>();
-    const originalTokens = result.originalTokens;
+    const originalTokens = data.originalTokens;
     const originalRealTopk = extractRealTopkFromTokens(originalTokens);
-    const originalToMergedMap = result.originalToMergedMap;
-    const mergedTokens = result.mergedTokens;
-    
-    // 使用binIndex判断是否是最两侧的bin
-    const isFirstBin = binIndex === 0;  // 第一个bin：包含超出下界的值
-    const isLastBin = binIndex === no_bins - 1;  // 最后一个bin：包含超出上界的值
+    const originalToMergedMap = data.originalToMergedMap;
+    const mergedTokens = data.mergedTokens;
 
-    // 遍历原始 token，找到 surprisal 在范围内的 token
     for (let i = 0; i < originalTokens.length; i++) {
         const surprisal = calculateSurprisal(originalRealTopk[i][1]);
-        let inRange = false;
-
-        if (isFirstBin) {
-            // 第一个bin：包含所有 < x1 的值（自身bin + 超出下界的数据）
-            inRange = surprisal < x1;
-        } else if (isLastBin) {
-            // 最后一个bin：包含所有 >= x0 的值（自身bin + 超出上界的数据）
-            inRange = surprisal >= x0;
-        } else {
-            // 中间bins：正常范围
-            inRange = surprisal >= x0 && surprisal < x1;
-        }
-        
-        if (inRange) {
-            // 映射到 merged token 索引
-            const mappedIndex = originalToMergedMap[i];
-            if (Number.isInteger(mappedIndex) && mappedIndex >= 0 && mappedIndex < mergedTokens.length) {
-                highlightedIndices.add(mappedIndex);
-            }
+        if (!Number.isFinite(surprisal)) continue;
+        if (!valueInBinRange(surprisal, x0, x1, binIndex, no_bins)) continue;
+        const mappedIndex = originalToMergedMap[i];
+        if (Number.isInteger(mappedIndex) && mappedIndex >= 0 && mappedIndex < mergedTokens.length) {
+            highlightedIndices.add(mappedIndex);
         }
     }
-
     return highlightedIndices;
 }
 
@@ -62,7 +54,7 @@ export function calculateTokenSurprisalHighlights(
  * @param x1 bin 结束值
  * @param binIndex bin在bins数组中的索引
  * @param no_bins 直方图的总bin数量
- * @param result 前端分析结果（包含 mergedTokens）
+ * @param data 前端分析结果（包含 mergedTokens）
  * @returns 需要高亮的 merged token 索引集合
  */
 export function calculateByteSurprisalHighlights(
@@ -70,95 +62,53 @@ export function calculateByteSurprisalHighlights(
     x1: number,
     binIndex: number,
     no_bins: number,
-    result: FrontendAnalyzeResult
+    data: HighlightData
 ): Set<number> {
     const highlightedIndices = new Set<number>();
-    const mergedTokens = result.mergedTokens;
-    
-    // 使用binIndex判断是否是最两侧的bin
-    const isFirstBin = binIndex === 0;  // 第一个bin：包含超出下界的值
-    const isLastBin = binIndex === no_bins - 1;  // 最后一个bin：包含超出上界的值
+    const mergedTokens = data.mergedTokens;
+    if (!mergedTokens?.length) return highlightedIndices;
 
-    // 遍历 merged token，找到信息密度在范围内的 token
     for (let i = 0; i < mergedTokens.length; i++) {
         const informationDensity = calculateSurprisalDensity(mergedTokens[i]);
-        let inRange = false;
-
-        if (isFirstBin) {
-            // 第一个bin：包含所有 < x1 的值（自身bin + 超出下界的数据）
-            inRange = informationDensity < x1;
-        } else if (isLastBin) {
-            // 最后一个bin：包含所有 >= x0 的值（自身bin + 超出上界的数据）
-            inRange = informationDensity >= x0;
-        } else {
-            // 中间bins：正常范围
-            inRange = informationDensity >= x0 && informationDensity < x1;
-        }
-        
-        if (inRange) {
-            highlightedIndices.add(i);
-        }
+        if (!Number.isFinite(informationDensity)) continue;
+        if (!valueInBinRange(informationDensity, x0, x1, binIndex, no_bins)) continue;
+        highlightedIndices.add(i);
     }
-
     return highlightedIndices;
 }
 
 /**
- * 根据直方图 bin 的范围计算需要高亮的 token 索引集合（基于 match_score）
- * 使用 attentionScores（与 mergedTokens 对齐），按 bin 范围筛选
+ * 根据直方图 bin 的范围计算需要高亮的 token 索引集合（基于 raw_score_normed）
+ * 使用 rawScoresNormed（与 mergedTokens 对齐），按 bin 范围筛选
  */
-export function calculateSemanticScoreHighlights(
+export function calculateRawScoreNormedHighlights(
     x0: number,
     x1: number,
     binIndex: number,
     no_bins: number,
-    result: FrontendAnalyzeResult & { attentionScores?: number[] }
+    data: HighlightData
 ): Set<number> {
     const highlightedIndices = new Set<number>();
-    const scores = result.attentionScores;
-    if (!scores || scores.length === 0) {
-        return highlightedIndices;
-    }
-
-    const isFirstBin = binIndex === 0;
-    const isLastBin = binIndex === no_bins - 1;
+    const scores = data.rawScoresNormed;
+    if (!scores?.length) return highlightedIndices;
 
     for (let i = 0; i < scores.length; i++) {
         const score = scores[i];
-        if (!Number.isFinite(score)) {
-            continue;
-        }
-
-        let inRange = false;
-        if (isFirstBin) {
-            inRange = score < x1;
-        } else if (isLastBin) {
-            inRange = score >= x0;
-        } else {
-            inRange = score >= x0 && score < x1;
-        }
-
-        if (inRange) {
-            highlightedIndices.add(i);
-        }
+        if (!Number.isFinite(score)) continue;
+        if (!valueInBinRange(score, x0, x1, binIndex, no_bins)) continue;
+        highlightedIndices.add(i);
     }
-
     return highlightedIndices;
 }
 
-/**
- * 直方图类型
- */
-export type HistogramType = 'token' | 'byte' | 'semantic';
-
 /**
  * 根据直方图类型和 bin 范围计算需要高亮的 token 索引集合
- * @param histogramType 直方图类型（'token' 或 'byte'）
+ * @param histogramType 直方图类型
  * @param x0 bin 起始值
  * @param x1 bin 结束值
  * @param binIndex bin在bins数组中的索引
  * @param no_bins 直方图的总bin数量
- * @param result 前端分析结果
+ * @param data 前端分析结果
  * @returns 需要高亮的 merged token 索引集合和对应的高亮样式
  */
 export function calculateHighlights(
@@ -167,22 +117,22 @@ export function calculateHighlights(
     x1: number,
     binIndex: number,
     no_bins: number,
-    result: FrontendAnalyzeResult
+    data: HighlightData
 ): { indices: Set<number>; style: 'border' | 'underline' } {
     if (histogramType === 'byte') {
         return {
-            indices: calculateByteSurprisalHighlights(x0, x1, binIndex, no_bins, result),
+            indices: calculateByteSurprisalHighlights(x0, x1, binIndex, no_bins, data),
             style: 'underline'
         };
     }
-    if (histogramType === 'semantic') {
+    if (histogramType === 'raw_score_normed') {
         return {
-            indices: calculateSemanticScoreHighlights(x0, x1, binIndex, no_bins, result as FrontendAnalyzeResult & { attentionScores?: number[] }),
+            indices: calculateRawScoreNormedHighlights(x0, x1, binIndex, no_bins, data),
             style: 'underline'
         };
     }
     return {
-        indices: calculateTokenSurprisalHighlights(x0, x1, binIndex, no_bins, result),
+        indices: calculateTokenSurprisalHighlights(x0, x1, binIndex, no_bins, data),
         style: 'border'
     };
 }

client/src/ts/utils/lognormalFit.ts

@@ -0,0 +1,144 @@
+/**
+ * 对数正态噪声拟合（纯数学，无依赖）
+ * 供 visualizationUpdater 使用，可独立在 Node 中测试
+ */
+
+export const LN_EPS = 1e-10;
+
+/** 标准正态 CDF Φ(x)，Abramowitz & Stegun 26.2.17 近似 */
+export function normCdf(x: number): number {
+    if (x <= -6) return 0;
+    if (x >= 6) return 1;
+    const a1 = 0.254829592, a2 = -0.284496736, a3 = 1.421413741, a4 = -1.453152027, a5 = 1.061405429, p = 0.3275911;
+    const sign = x < 0 ? -1 : 1;
+    const t = 1 / (1 + p * Math.abs(x) / Math.SQRT2);
+    const y = 1 - (((((a5 * t + a4) * t + a3) * t + a2) * t + a1) * t) * Math.exp(-x * x / 2);
+    return 0.5 * (1 + sign * y);
+}
+
+/** 对数正态 CDF：F(x) = Φ((log(x) - μ) / σ)，x > 0 */
+export function logNormalCdf(x: number, mu: number, sigma: number): number {
+    if (x <= 0) return 0;
+    return normCdf((Math.log(x) - mu) / sigma);
+}
+
+/** 区间 [a, b) 在 log-normal(μ,σ) 下的期望计数：n × (CDF(b) - CDF(a)) */
+export function logNormalExpectedCountInInterval(
+    a: number, b: number, n: number, mu: number, sigma: number
+): number {
+    return n * (logNormalCdf(b, mu, sigma) - logNormalCdf(a, mu, sigma));
+}
+
+/** 对数正态 PDF：f(x) = φ((log(x)-μ)/σ) / (xσ)，x > 0 */
+export function logNormalPdf(x: number, mu: number, sigma: number): number {
+    if (x <= 0 || sigma <= 0) return 0;
+    const z = (Math.log(x) - mu) / sigma;
+    return normPdf(z) / (x * sigma);
+}
+
+/** 标准正态 PDF φ(x) */
+function normPdf(x: number): number {
+    return Math.exp(-x * x / 2) / Math.sqrt(2 * Math.PI);
+}
+
+/** 逆 Mills 比率 λ(α) = φ(α)/Φ(α)，α → −∞ 时近似 |α| */
+function millsRatio(alpha: number): number {
+    const Phi = normCdf(alpha);
+    if (Phi < 1e-300) return Math.abs(alpha);
+    return normPdf(alpha) / Phi;
+}
+
+/**
+ * 截尾对数正态 MLE（右截尾于 τ）
+ * 导出供测试对比 tau=max(samples) vs tau=固定值
+ */
+export function fitLogNormalTruncatedMLE(
+    noiseScores: number[],
+    tau: number
+): { mu: number; sigma: number } | null {
+    const n = noiseScores.length;
+    if (n < 2 || tau <= LN_EPS) return null;
+
+    const T = Math.log(tau);
+    const logData = noiseScores.map(x => Math.log(x));
+    const ybar = logData.reduce((a, b) => a + b, 0) / n;
+    const s2 = logData.reduce((a, x) => a + (x - ybar) ** 2, 0) / n;
+    const s = Math.sqrt(s2);
+    if (s <= 0 || !isFinite(s)) return null;
+
+    const delta = T - ybar;
+
+    const F = (alpha: number): number => {
+        const lam = millsRatio(alpha);
+        if (!isFinite(lam)) return delta > 0 ? -1 : 1;
+        const g = alpha + lam;
+        const h = 1 - lam * g;
+        if (h <= 0) return NaN;
+        return g - (delta / s) * Math.sqrt(h);
+    };
+
+    const lo0 = -8, hi0 = delta / s + 8;
+    const Flo = F(lo0), Fhi = F(hi0);
+    if (!isFinite(Flo) || !isFinite(Fhi) || Flo * Fhi > 0) return null;
+
+    let lo = lo0, hi = hi0, Flo_cur = Flo;
+    for (let i = 0; i < 60; i++) {
+        const mid = (lo + hi) / 2;
+        const Fmid = F(mid);
+        if (!isFinite(Fmid) || (hi - lo) < 1e-12) break;
+        if (Flo_cur * Fmid <= 0) { hi = mid; }
+        else { lo = mid; Flo_cur = Fmid; }
+    }
+
+    const alpha = (lo + hi) / 2;
+    const lam = millsRatio(alpha);
+    if (!isFinite(lam)) return null;
+    const h = 1 - lam * (alpha + lam);
+    if (h <= 0) return null;
+
+    const sigma = s / Math.sqrt(h);
+    const mu = ybar + sigma * lam;
+    if (!isFinite(sigma) || sigma <= 0 || !isFinite(mu)) return null;
+    return { mu, sigma };
+}
+
+/*
+ * todo: 未知原因的偏差现象：
+ * Monte Carlo 下 E[μ̂] 随 n 减小单调增大（系统性正偏），而非围绕真值的随机波动。
+> inforadar@0.1.0 test:lognormal:tau
+> npx tsx ts/utils/lognormalFit.tauBoundary.test.ts
+=== 截尾对数正态拟合硬指标测试 ===
+
+真实参数: μ=-2, σ=0.8, τ=1
+Monte Carlo 500 次，fitLogNormalNoiseExpectedCounts percentile=0.9
+
+n      | E[μ̂]    E[σ̂]    Δμ      Δσ
+-------|------------------------------
+ 1600 | -1.9977  0.8013  0.0023  0.0013
+  800 | -1.9950  0.8023  0.0050  0.0023
+  400 | -1.9910  0.8054  0.0090  0.0054
+  200 | -1.9851  0.8059  0.0149  0.0059
+  100 | -1.9722  0.8096  0.0278  0.0096
+   50 | -1.9541  0.8056  0.0459  0.0056
+ */
+
+/**
+ * 从 (μ, σ) 计算直方图各 bin 的期望计数
+ */
+export function computeExpectedCounts(
+    mu: number,
+    sigma: number,
+    extent: [number, number],
+    noBins: number,
+    n: number
+): number[] {
+    const binWidth = (extent[1] - extent[0]) / noBins;
+    const expectedCounts: number[] = [];
+    for (let i = 0; i < noBins; i++) {
+        const a = extent[0] + i * binWidth;
+        const b = extent[0] + (i + 1) * binWidth;
+        const p = logNormalCdf(b, mu, sigma) - logNormalCdf(a, mu, sigma);
+        expectedCounts.push(n * p);
+    }
+    return expectedCounts;
+}

client/src/ts/utils/queryHistory.ts

@@ -41,10 +41,12 @@ export interface InitQueryHistoryDropdownOptions {
     dropdownId: string;
     onSelect: () => void;
     onHistorySelect?: () => void;
+    /** 删除某条历史时回调，用于同步清理相关缓存 */
+    onRemove?: (query: string) => void;
 }
 
 export function initQueryHistoryDropdown(options: InitQueryHistoryDropdownOptions): void {
-    const { input, dropdownId, onSelect, onHistorySelect } = options;
+    const { input, dropdownId, onSelect, onHistorySelect, onRemove } = options;
     if (!input) return;
 
     const wrapper = input.closest('.semantic-search-input-wrapper');
@@ -82,6 +84,7 @@ export function initQueryHistoryDropdown(options: InitQueryHistoryDropdownOption
             btn.onclick = (e) => {
                 e.stopPropagation();
                 remove(q);
+                onRemove?.(q);
                 render();
             };
             li.appendChild(span);

client/src/ts/utils/semanticResultCache.ts

@@ -0,0 +1,104 @@
+/**
+ * 语义分析结果缓存：以 text + query + submode 的 hash 为索引，最大 100 条。
+ * 持久化到 localStorage，刷新后保留。删除查询历史时需调用 removeByQuery 清理对应缓存。
+ */
+
+const MAX_SIZE = 100;
+const STORAGE_KEY = 'info_radar_semantic_result_cache';
+
+export type SemanticCacheResult = {
+    success: boolean;
+    model?: string;
+    token_attention?: Array<{ offset: [number, number]; raw: string; score: number }>;
+    debug_info?: { abbrev?: string; topk_tokens?: string[]; topk_probs?: number[] };
+    full_match_degree?: number;
+    message?: string;
+};
+
+type StoredEntry = SemanticCacheResult & { _query?: string };
+
+function simpleHash(s: string): string {
+    let h = 0;
+    for (let i = 0; i < s.length; i++) {
+        h = ((h << 5) - h + s.charCodeAt(i)) | 0;
+    }
+    return (h >>> 0).toString(36);
+}
+
+function buildKey(text: string, query: string, submode?: string): string {
+    const parts = [text, query, submode ?? ''];
+    return simpleHash(parts.join('\0'));
+}
+
+const cache = new Map<string, StoredEntry>();
+let keyOrder: string[] = [];
+
+function load(): void {
+    try {
+        const raw = localStorage.getItem(STORAGE_KEY);
+        if (!raw) return;
+        const parsed = JSON.parse(raw) as { entries?: Record<string, StoredEntry>; keyOrder?: string[] };
+        if (!parsed?.entries || typeof parsed.entries !== 'object') return;
+        cache.clear();
+        for (const [k, v] of Object.entries(parsed.entries)) {
+            if (v && typeof v === 'object') cache.set(k, v);
+        }
+        keyOrder = Array.isArray(parsed.keyOrder)
+            ? parsed.keyOrder.filter((k) => cache.has(k)).slice(-MAX_SIZE)
+            : [...cache.keys()];
+    } catch {
+        cache.clear();
+        keyOrder = [];
+    }
+}
+
+load();
+
+function persist(): void {
+    try {
+        const entries: Record<string, StoredEntry> = {};
+        for (const [k, v] of cache) entries[k] = v;
+        localStorage.setItem(STORAGE_KEY, JSON.stringify({ entries, keyOrder }));
+    } catch {
+        // QuotaExceededError 等，忽略
+    }
+}
+
+function evictOne(): void {
+    if (keyOrder.length < MAX_SIZE) return;
+    const oldest = keyOrder.shift()!;
+    cache.delete(oldest);
+}
+
+export function get(text: string, query: string, submode?: string): SemanticCacheResult | undefined {
+    const key = buildKey(text, query, submode);
+    const entry = cache.get(key);
+    if (!entry) return undefined;
+    const { _query, ...rest } = entry as SemanticCacheResult & { _query?: string };
+    return rest;
+}
+
+export function set(text: string, query: string, result: SemanticCacheResult, submode?: string): void {
+    const key = buildKey(text, query, submode);
+    if (cache.has(key)) {
+        const idx = keyOrder.indexOf(key);
+        if (idx >= 0) keyOrder.splice(idx, 1);
+    }
+    evictOne();
+    cache.set(key, { ...result, _query: query });
+    keyOrder.push(key);
+    persist();
+}
+
+export function removeByQuery(query: string): void {
+    const keysToRemove: string[] = [];
+    for (const [key, entry] of cache) {
+        if (entry._query === query) keysToRemove.push(key);
+    }
+    for (const key of keysToRemove) {
+        cache.delete(key);
+        const idx = keyOrder.indexOf(key);
+        if (idx >= 0) keyOrder.splice(idx, 1);
+    }
+    if (keysToRemove.length) persist();
+}

client/src/ts/utils/signalThresholdDetector.ts

@@ -0,0 +1,330 @@
+/**
+ * 信号阈值检测：自动找到「噪声/信号」边界
+ *
+ * 输入：raw score normed [0,1]
+ * 输出：{ threshold, confidence, mu, sigma }；无命中时返回 null
+ *
+ * 算法概要：
+ * 1. 迭代 0：用全部样本（P0=1）拟合截尾对数正态 (μ, σ)，从 startPercentile 分位 bin 起逐 bin 扫描
+ *    - 每个 bin [τ_left, τ_right) 左闭右开：obsInBin = 该 bin 内观测计数，expInBin = n × (CDF(τ_right) - CDF(τ_left))
+ *    - 纯噪声区：信号样本不在 bin 内 → excess ≈ 0
+ *    - 到信号边界：bin 内出现超额样本 → excess 跃升
+ *    - 不重叠扫描：bin 边界取相邻点中点，τ_right >= τ_left + MIN_BIN_WIDTH，obsInBin >= MIN_OBSERVED
+ *    - 误报概率：cumulativeFalsePositiveProbability = ∏(1-Φ(excess_i))，excess>excessMin 时累积，否则重置
+ *    - 当 cumulativeFalsePositiveProbability <= 1-CONFIDENCE_THRESHOLD 时，取首次命中 bin 的左边界 sorted[j] 为阈值（保守）
+ *    - 若全程无命中，返回 null
+ * 2. 迭代 1..N：用 threshold 以下样本重拟合，再扫描；阈值变化不大则提前结束；任一迭代无命中则返回 null
+ *
+ * 与现有 lognormalFit 逻辑独立，未来可能替换现有拟合代码
+ */
+
+import { fitLogNormalTruncatedMLE, logNormalExpectedCountInInterval, normCdf, LN_EPS } from './lognormalFit';
+import { computeFitQuality } from './fitQuality';
+
+/** 置信度阈值，达到此值即判定「确定找到」信号边界；默认 0.9999 */
+const CONFIDENCE_THRESHOLD = 0.9999;
+/** excess 最小阈值，排除无意义随机波动；需 excess > 此值才计为命中 */
+const EXCESS_MIN = 0.1;
+const MIN_OBSERVED = 1; // 每个 bin 至少 N 个观测
+const MIN_BIN_WIDTH = 0.01; // bin 最小宽度；边界取相邻点中点
+const MIN_SAMPLE_SIZE = 20;
+const P0 = 1; // 迭代初始的样本拟合比例
+const MAX_REFINE_ITER = 10;
+const THRESHOLD_CONVERGE_EPS = 0.01; //迭代收敛阈值
+/** 扫描起始分位，默认 0.5（从 50% 分位所在 bin 开始） */
+const START_PERCENTILE_DEFAULT = 0.5;
+/** 输出过滤：最终 confidence 低于此值则返回 null；与 CONFIDENCE_THRESHOLD 不同，后者用于内部扫描判定 */
+const MIN_OUTPUT_CONFIDENCE = 0.9;
+/** expInBin 最小有效值，避免除零或数值不稳定 */
+const EXP_IN_BIN_EPS = 1e-10;
+
+/** 内部：evaluateBins 的中间结果，仅 threshold + confidence */
+interface SignalThresholdScanResult {
+    threshold: number;
+    confidence: number;
+}
+
+/** 对外：findSignalThreshold 成功时的完整结果，mu/sigma/bins 必存在 */
+export interface SignalThresholdResult {
+    threshold: number;
+    /** 0~1，统计置信度：有命中时 1-误报概率 */
+    confidence: number;
+    /** 最终拟合的 μ（供 histogram 使用） */
+    mu: number;
+    /** 最终拟合的 σ（供 histogram 使用） */
+    sigma: number;
+    /** 全范围 bins（供 signal prob 等使用） */
+    bins: SignalThresholdBin[];
+}
+
+export interface SignalThresholdBin {
+    tauLeft: number;
+    tauRight: number;
+    obsInBin: number;
+    expInBin: number;
+}
+
+/** 内部：bin 结构（tauLeft/tauRight/obsInBin）仅依赖 sorted，迭代间不变 */
+interface BinStructure {
+    tauLeft: number;
+    tauRight: number;
+    obsInBin: number;
+}
+
+const TAU_RIGHT_EPSILON = 1e-6;
+
+const PERCENTILE_DIAGNOSTICS = [0.5, 0.6, 0.7, 0.8, 0.9, 0.95, 0.99, 1] as const;
+
+/** 计算 excess = (obs - exp) / sqrt(exp)，exp 过小时避免除零 */
+function computeExcess(obsInBin: number, expInBin: number): number {
+    if (expInBin <= EXP_IN_BIN_EPS) return obsInBin > 0 ? Infinity : 0;
+    return (obsInBin - expInBin) / Math.sqrt(expInBin);
+}
+
+/** 打印不同分位数下的拟合结果，用于验证渐近一致性 */
+function logPercentileDiagnostics(scores: number[]): void {
+    const sorted = [...scores].sort((a, b) => a - b);
+    const n = sorted.length;
+    if (n < 2) return;
+    const rows: Array<{ p: number; n: number; mu: number; sigma: number }> = [];
+    for (const p of PERCENTILE_DIAGNOSTICS) {
+        const pIdx = Math.max(1, Math.min(n, Math.round(n * p)));
+        const noiseNorm = sorted.slice(0, pIdx);
+        const tau = pIdx < n ? (sorted[pIdx - 1]! + sorted[pIdx]!) / 2 : sorted[pIdx - 1]!;
+        const fit = fitLogNormalTruncatedMLE(noiseNorm, tau);
+        if (fit) rows.push({ p, n: pIdx, mu: fit.mu, sigma: fit.sigma });
+    }
+    if (rows.length === 0) return;
+    console.log('[signalThreshold] 渐近一致性诊断 (percentile → μ, σ)');
+    for (const { p, n, mu, sigma } of rows) {
+        console.log(`  p=${p} n=${n}: μ=${mu.toFixed(4)}, σ=${sigma.toFixed(4)}`);
+    }
+}
+/** verbose 时打印完整 bin 扫描日志（独立于 evaluateBins，仅追加输出） */
+function printBinScanLogs(bins: SignalThresholdBin[], excessMin: number): void {
+    console.log('[signalThreshold] 完整扫描明细 τ_left | τ_right | obsInBin | expInBin | excess | binConf | hit | confidence');
+    let cumulativeFalsePositiveProbability = 1;
+    let firstHitTauLeft: number | null = null;
+    for (const bin of bins) {
+        const excess = computeExcess(bin.obsInBin, bin.expInBin);
+        const hit = excess > excessMin;
+        const binConfidence = normCdf(excess);
+        if (hit) {
+            if (firstHitTauLeft === null) firstHitTauLeft = bin.tauLeft;
+            cumulativeFalsePositiveProbability *= 1 - binConfidence;
+            const confidence = 1 - cumulativeFalsePositiveProbability;
+            console.log(`[signalThreshold]   ${bin.tauLeft.toFixed(4)} | ${bin.tauRight.toFixed(4)} | ${String(bin.obsInBin).padStart(7)} | ${bin.expInBin.toFixed(1).padStart(8)} | ${excess.toFixed(2).padStart(6)} | ${binConfidence.toFixed(4)} | ✓ | ${confidence.toFixed(4)}`);
+        } else {
+            cumulativeFalsePositiveProbability = 1;
+            firstHitTauLeft = null;
+            console.log(`[signalThreshold]   ${bin.tauLeft.toFixed(4)} | ${bin.tauRight.toFixed(4)} | ${String(bin.obsInBin).padStart(7)} | ${bin.expInBin.toFixed(1).padStart(8)} | ${excess.toFixed(2).padStart(6)} | ${binConfidence.toFixed(4)} |   | -`);
+        }
+    }
+}
+
+/** bin 边界取相邻点中点，τ_right >= τ_left + MIN_BIN_WIDTH，obsInBin >= MIN_OBSERVED；仅依赖 sorted，迭代间不变 */
+function formBinStructures(sorted: number[]): BinStructure[] {
+    const n = sorted.length;
+    const mids: number[] = [];
+    for (let i = 0; i < n - 1; i++) mids.push((sorted[i]! + sorted[i + 1]!) / 2);
+    const structures: BinStructure[] = [];
+    let tauLeft = sorted[0]! - TAU_RIGHT_EPSILON;
+
+    while (tauLeft < sorted[n - 1]!) {
+        let midIdx = mids.findIndex((m) => m >= tauLeft + MIN_BIN_WIDTH);
+        let tauRight = midIdx >= 0 ? mids[midIdx]! : sorted[n - 1]! + TAU_RIGHT_EPSILON;
+
+        let leftIdx = sorted.findIndex((v) => v >= tauLeft);
+        let rightIdx = midIdx >= 0 ? sorted.findIndex((v) => v >= tauRight) : -1;
+        let obsInBin = leftIdx < 0 ? 0 : rightIdx < 0 ? n - leftIdx : rightIdx - leftIdx;
+
+        while (obsInBin < MIN_OBSERVED && midIdx >= 0 && midIdx < mids.length - 1) {
+            midIdx++;
+            tauRight = mids[midIdx]!;
+            rightIdx = sorted.findIndex((v) => v >= tauRight);
+            obsInBin = leftIdx < 0 ? 0 : rightIdx < 0 ? n - leftIdx : rightIdx - leftIdx;
+        }
+        if (obsInBin < MIN_OBSERVED) {
+            tauRight = sorted[n - 1]! + TAU_RIGHT_EPSILON;
+            rightIdx = -1;
+            obsInBin = leftIdx < 0 ? 0 : n - leftIdx;
+            if (obsInBin < MIN_OBSERVED) break;
+        }
+
+        structures.push({ tauLeft, tauRight, obsInBin });
+        tauLeft = tauRight;
+        if (tauRight >= sorted[n - 1]! + TAU_RIGHT_EPSILON) break;
+    }
+    return structures;
+}
+
+/** 遍历 bin 结构，按需计算 expInBin，返回阈值结果；通过 obsInBin 累积找到 startPercentile 分位对应 bin，从该 bin 开始扫描 */
+function evaluateBins(
+    structures: BinStructure[],
+    n: number,
+    mu: number,
+    sigma: number,
+    excessMin: number,
+    confidenceThreshold: number,
+    verbose: boolean,
+    startPercentile: number
+): SignalThresholdScanResult | null {
+    let cumulativeFalsePositiveProbability = 1;
+    let firstHitTauLeft: number | null = null;
+
+    const K = Math.min(Math.floor((n - 1) * startPercentile), n - 1);
+    let cumSum = 0;
+    let startIdx = 0;
+    for (let i = 0; i < structures.length; i++) {
+        if (K < cumSum + structures[i]!.obsInBin) {
+            startIdx = i;
+            break;
+        }
+        cumSum += structures[i]!.obsInBin;
+    }
+    const structuresToScan = structures.slice(startIdx);
+
+    if (verbose) {
+        console.log('[signalThreshold] 扫描明细 τ_left | τ_right | obsInBin | expInBin | excess | binConf | hit | confidence');
+    }
+
+    for (const s of structuresToScan) {
+        const expInBin = logNormalExpectedCountInInterval(s.tauLeft, s.tauRight, n, mu, sigma);
+        const excess = computeExcess(s.obsInBin, expInBin);
+        const hit = excess > excessMin;
+        const binConfidence = normCdf(excess);
+
+        if (hit) {
+            if (firstHitTauLeft === null) firstHitTauLeft = s.tauLeft;
+            cumulativeFalsePositiveProbability *= 1 - binConfidence;
+            const confidence = 1 - cumulativeFalsePositiveProbability;
+            if (verbose) {
+                console.log(`[signalThreshold]   ${s.tauLeft.toFixed(4)} | ${s.tauRight.toFixed(4)} | ${String(s.obsInBin).padStart(7)} | ${expInBin.toFixed(1).padStart(8)} | ${excess.toFixed(2).padStart(6)} | ${binConfidence.toFixed(4)} | ✓ | ${confidence.toFixed(4)}`);
+            }
+            if (confidence >= confidenceThreshold) {
+                return { threshold: firstHitTauLeft, confidence };
+            }
+        } else {
+            cumulativeFalsePositiveProbability = 1;
+            firstHitTauLeft = null;
+            if (verbose) {
+                console.log(`[signalThreshold]   ${s.tauLeft.toFixed(4)} | ${s.tauRight.toFixed(4)} | ${String(s.obsInBin).padStart(7)} | ${expInBin.toFixed(1).padStart(8)} | ${excess.toFixed(2).padStart(6)} | ${binConfidence.toFixed(4)} |   | -`);
+            }
+        }
+    }
+
+    if (firstHitTauLeft !== null) {
+        return { threshold: firstHitTauLeft, confidence: 1 - cumulativeFalsePositiveProbability };
+    }
+    return null;
+}
+
+/**
+ * 从 raw score normed 数组自动检测信号阈值
+ * @param rawScoresNormed 归一化分数 [0,1]
+ * @param verbose 是否输出详细日志，默认 false
+ * @returns 成功时返回完整结果 { threshold, confidence, mu, sigma, bins }；样本不足、拟合失败或无命中时返回 null
+ */
+export function findSignalThreshold(
+    rawScoresNormed: number[],
+    verbose = false
+): SignalThresholdResult | null {
+    const values = rawScoresNormed.filter(
+        (s) => typeof s === 'number' && isFinite(s) && s > LN_EPS
+    );
+    const sorted = [...values].sort((a, b) => a - b);
+    const n = sorted.length;
+
+    if (n < MIN_SAMPLE_SIZE) {
+        if (verbose) console.log('[signalThreshold] 样本不足 n<', MIN_SAMPLE_SIZE, '，跳过');
+        return null;
+    }
+
+    const p0 = P0;
+    const splitIdx = Math.max(1, Math.min(n, Math.round(n * p0)));
+    if (verbose) console.log('[signalThreshold] n=', n, 'splitIdx=', splitIdx);
+
+    let result: SignalThresholdScanResult | null = null;
+    let lastFit = { mu: 0, sigma: 0 };
+    const binStructures = formBinStructures(sorted);
+
+    for (let iter = 0; iter <= MAX_REFINE_ITER; iter++) {
+        if (iter > 0 && result === null) return null;
+        const thresholdForNoise = result?.threshold ?? 0;
+        const noiseSamples = iter === 0
+            ? sorted.slice(0, splitIdx)
+            : sorted.filter((x) => x <= thresholdForNoise);
+        const tauBoundary = iter === 0
+            ? (splitIdx < n ? (sorted[splitIdx - 1]! + sorted[splitIdx]!) / 2 : sorted[splitIdx - 1]!)
+            : thresholdForNoise;
+
+        if (iter > 0 && noiseSamples.length < MIN_SAMPLE_SIZE) {
+            if (verbose) console.log('[signalThreshold] 迭代', iter, '提前结束：噪声样本数<', MIN_SAMPLE_SIZE);
+            return null;
+        }
+
+        if (verbose && iter === 0) {
+            const nInit = noiseSamples.length;
+            const minN = noiseSamples[0]!, maxN = noiseSamples[nInit - 1]!;
+            const midN = noiseSamples[Math.floor(nInit / 2)]!;
+            console.log('[signalThreshold] 迭代 0 噪声样本 n=', nInit, 'min=', minN.toFixed(4), 'max=', maxN.toFixed(4), 'median=', midN.toFixed(4));
+        }
+
+        const fit = fitLogNormalTruncatedMLE(noiseSamples, tauBoundary);
+        if (fit === null) {
+            if (verbose) console.log('[signalThreshold] 迭代', iter, '拟合失败');
+            return null;
+        }
+        lastFit = { mu: fit.mu, sigma: fit.sigma };
+
+        const q = computeFitQuality(noiseSamples, tauBoundary, fit.mu, fit.sigma);
+        if (verbose) {
+            console.log('[signalThreshold] 迭代', iter, '拟合 μ=', fit.mu.toFixed(4), 'σ=', fit.sigma.toFixed(4), '| maxDiff=', q.maxDiff.toFixed(4), 'RMSE=', q.rmse.toFixed(4));
+            if (iter === 0) {
+                console.log('[signalThreshold] 迭代', iter, '从', (START_PERCENTILE_DEFAULT * 100).toFixed(0), '% 分位 bin 开始扫描 (excess>', EXCESS_MIN, ', confidence>=', CONFIDENCE_THRESHOLD, ')');
+            }
+        }
+
+        const scanResult = evaluateBins(binStructures, n, fit.mu, fit.sigma, EXCESS_MIN, CONFIDENCE_THRESHOLD, verbose, START_PERCENTILE_DEFAULT);
+        if (scanResult === null) {
+            if (verbose) console.log('[signalThreshold] 迭代', iter, '未检测到阈值');
+            return null;
+        }
+
+        const savedThreshold = result?.threshold;
+        result = scanResult;
+
+        if (iter > 0 && savedThreshold !== undefined) {
+            const delta = Math.abs(result.threshold - savedThreshold);
+            if (verbose) {
+                console.log('[signalThreshold] 迭代', iter, '新阈值=', result.threshold.toFixed(4), 'confidence=', result.confidence.toFixed(2), 'delta=', delta.toFixed(6));
+            }
+            if (delta < THRESHOLD_CONVERGE_EPS) {
+                if (verbose) console.log('[signalThreshold] 迭代', iter, '收敛，最终阈值=', result.threshold.toFixed(4));
+                break;
+            }
+            if (iter === MAX_REFINE_ITER && verbose) {
+                console.log('[signalThreshold] 达到最大迭代次数，最终阈值=', result.threshold.toFixed(4));
+            }
+        } else if (verbose) {
+            console.log('[signalThreshold] 迭代 0 检测到阈值', result.threshold.toFixed(4), 'confidence=', result.confidence.toFixed(2));
+        }
+    }
+
+    const bins: SignalThresholdBin[] = binStructures.map((s) => ({
+        ...s,
+        expInBin: logNormalExpectedCountInInterval(s.tauLeft, s.tauRight, n, lastFit.mu, lastFit.sigma),
+    }));
+    if (verbose && bins.length > 0) {
+        printBinScanLogs(bins, EXCESS_MIN);
+        logPercentileDiagnostics(values);
+    }
+    if (result === null) return null;
+    if (result.confidence < MIN_OUTPUT_CONFIDENCE) {
+        if (verbose) {
+            console.warn('[signalThreshold] confidence <', (MIN_OUTPUT_CONFIDENCE * 100).toFixed(0), '%，返回 null。当前 confidence=', result.confidence.toFixed(2));
+        }
+        return null;
+    }
+    return { ...result, mu: lastFit.mu, sigma: lastFit.sigma, bins };
+}

client/src/ts/utils/tokenDisplayUtils.ts

@@ -0,0 +1,97 @@
+/**
+ * Token 文本显示工具：特殊字符可视化、HTML 转义
+ * 与 Tooltip、TopK 图表等共享
+ */
+
+function escapeHtmlImpl(text: string): string {
+    const div = document.createElement('div');
+    div.textContent = text;
+    return div.innerHTML;
+}
+
+function isWhitespaceChar(char: string): boolean {
+    return /\p{White_Space}/u.test(char);
+}
+
+function isPrintableChar(char: string): boolean {
+    if (isWhitespaceChar(char)) return false;
+    const codePoint = char.codePointAt(0);
+    if (codePoint === undefined) return false;
+    if (codePoint >= 32 && codePoint <= 126) return true;
+    if (
+        (codePoint >= 0x00A0 && codePoint <= 0x00FF) ||
+        (codePoint >= 0x0100 && codePoint <= 0x017F) ||
+        (codePoint >= 0x0180 && codePoint <= 0x024F) ||
+        (codePoint >= 0x2000 && codePoint <= 0x206F) ||
+        (codePoint >= 0x2070 && codePoint <= 0x209F) ||
+        (codePoint >= 0x20A0 && codePoint <= 0x20CF) ||
+        (codePoint >= 0x2100 && codePoint <= 0x214F) ||
+        (codePoint >= 0x2190 && codePoint <= 0x21FF) ||
+        (codePoint >= 0x2200 && codePoint <= 0x22FF) ||
+        (codePoint >= 0x2300 && codePoint <= 0x23FF) ||
+        (codePoint >= 0x2400 && codePoint <= 0x243F) ||
+        (codePoint >= 0x2E00 && codePoint <= 0x2E7F) ||
+        (codePoint >= 0x3000 && codePoint <= 0x303F) ||
+        (codePoint >= 0x3040 && codePoint <= 0x309F) ||
+        (codePoint >= 0x30A0 && codePoint <= 0x30FF) ||
+        (codePoint >= 0x4E00 && codePoint <= 0x9FFF) ||
+        (codePoint >= 0xAC00 && codePoint <= 0xD7AF) ||
+        (codePoint >= 0xF900 && codePoint <= 0xFAFF) ||
+        (codePoint >= 0xFF00 && codePoint <= 0xFFEF)
+    ) return true;
+    return false;
+}
+
+function visualizeSpecialCharsImpl(text: string): string {
+    let result = text
+        .replace(/\r\n/g, '[CRLF]')
+        .replace(/\n/g, '[LF]')
+        .replace(/\r/g, '[CR]')
+        .replace(/\t/g, '[TAB]')
+        .replace(/\u3000/g, '[FS]')
+        .replace(/ /g, '·');
+
+    const processed: string[] = [];
+    let inBracket = false;
+
+    for (let i = 0; i < result.length; i++) {
+        const char = result[i];
+        if (char === '[') {
+            inBracket = true;
+            processed.push(char);
+        } else if (char === ']' && inBracket) {
+            processed.push(char);
+            inBracket = false;
+        } else if (inBracket) {
+            processed.push(char);
+        } else {
+            if (isPrintableChar(char)) {
+                processed.push(char);
+            } else {
+                const codePoint = char.codePointAt(0);
+                if (codePoint !== undefined) {
+                    const hexCode = codePoint.toString(16).toUpperCase().padStart(4, '0');
+                    processed.push(`[U+${hexCode}]`);
+                } else {
+                    processed.push(char);
+                }
+            }
+        }
+    }
+    return processed.join('');
+}
+
+/** 处理候选词文本，与主 token 保持一致：先可视化特殊字符，再 HTML 转义 */
+export function processCandidateText(text: string): string {
+    return escapeHtmlImpl(visualizeSpecialCharsImpl(text));
+}
+
+/** HTML 转义 */
+export function escapeHtml(text: string): string {
+    return escapeHtmlImpl(text);
+}
+
+/** 可视化特殊字符 */
+export function visualizeSpecialChars(text: string): string {
+    return visualizeSpecialCharsImpl(text);
+}

client/src/ts/utils/topkChartUtils.ts

@@ -0,0 +1,71 @@
+/**
+ * TopK 图表渲染工具：与 Tooltip 中的 topk 图表完全一致
+ * 供 Tooltip 和语义分析 debug info 复用
+ */
+
+import * as d3 from 'd3';
+import { processCandidateText } from './tokenDisplayUtils';
+
+const DISPLAY_TOPK = 10;
+/** Tooltip 默认条形宽度 */
+const MAX_BAR_WIDTH = 60;
+/** Semantic debug 专用：更大条形与列宽，tooltip 不受影响 */
+const SEMANTIC_DEBUG_MAX_BAR = 100;
+const SEMANTIC_DEBUG_BAR_CELL = 180;
+
+export interface TopkChartOptions {
+    /** 高亮的 token（与当前 token 一致时用 selectedColor） */
+    selectedToken?: string;
+    normalColor?: string;
+    selectedColor?: string;
+    /** 条形最大宽度 px */
+    maxBarWidth?: number;
+    /** 条形列单元格宽度 px */
+    barCellWidth?: number;
+    numFormat?: (n: number) => string;
+}
+
+function getThemeColors(): { normalColor: string; selectedColor: string } {
+    const isDark = document.documentElement.getAttribute('data-theme') === 'dark';
+    return {
+        normalColor: isDark ? '#ccc' : '#333',
+        selectedColor: isDark ? '#ff6666' : '#933',
+    };
+}
+
+/** 生成与 Tooltip 完全一致的 TopK 图表 HTML */
+export function renderTopkChartHtml(
+    data: Array<{ token: string; prob: number }>,
+    options?: TopkChartOptions
+): string {
+    if (!data.length) return '';
+
+    const { normalColor, selectedColor } = getThemeColors();
+    const norm = options?.normalColor ?? normalColor;
+    const sel = options?.selectedColor ?? selectedColor;
+    const maxBar = options?.maxBarWidth ?? MAX_BAR_WIDTH;
+    const numF = options?.numFormat ?? d3.format('.3f');
+
+    const maxProb = data[0]?.prob ?? 1;
+    const scale = d3.scaleLinear().domain([0, maxProb]).range([0, maxBar]);
+    const barCellW = options?.barCellWidth ?? 110;
+
+    const rows = data.slice(0, DISPLAY_TOPK).map((d) => {
+        const color = options?.selectedToken !== undefined && d.token === options.selectedToken ? sel : norm;
+        const bar = `<div style="display: table-cell; width:${barCellW}px;padding-left:5px;">` +
+            `<div style="display:inline-block;width: ${scale(d.prob)}px;background-color:${color};height: 10px;"></div>` +
+            ` <div style="display:inline-block;color: ${color};">${numF(d.prob)}</div></div>`;
+        const text = `<div style="display: table-cell;color: ${color};padding-right:5px;">${processCandidateText(d.token)}</div>`;
+        return `<div class="row" style="display: table-row;">${bar} ${text}</div>`;
+    });
+
+    return rows.join('');
+}
+
+/** 生成完整 TopK 图表 HTML（含容器），用于独立展示如 semantic debug。tooltip 用 renderTopkChartHtml，不传尺寸选项。 */
+export function renderTopkChartFullHtml(data: Array<{ token: string; prob: number }>, options?: TopkChartOptions): string {
+    const opts = options ?? {};
+    const semanticOpts = { ...opts, maxBarWidth: opts.maxBarWidth ?? SEMANTIC_DEBUG_MAX_BAR, barCellWidth: opts.barCellWidth ?? SEMANTIC_DEBUG_BAR_CELL };
+    const rows = renderTopkChartHtml(data, semanticOpts);
+    return rows ? `<div class="semantic-debug-topk-chart predictions predictions-table">${rows}</div>` : '';
+}

client/src/ts/utils/visualizationConfigs.ts

@@ -1,4 +1,7 @@
 import { tr } from '../lang/i18n-lite';
+import type { HistogramExtent, HistogramExtentBound } from '../vis/Histogram';
+
+export type { HistogramExtent, HistogramExtentBound };
 
 /**
  * 直方图基础配置类型
@@ -6,12 +9,14 @@ import { tr } from '../lang/i18n-lite';
 export interface HistogramBaseConfig {
   label: string;
   no_bins: number;
-  extent: [number, number];
-  averageLabel: string;
+  extent: HistogramExtent;
+  averageLabel?: string;
   showLeftInfinity?: boolean;
   showRightInfinity?: boolean;
   xAxisTickSkip?: number;
-  yScaleType?: 'linear' | 'sqrt';
+  /** x轴刻度凑整：true=仅显示 step 整数倍处的标签，false/undefined=显示全部 */
+  xAxisTickRound?: boolean;
+  yScaleType?: 'linear' | 'sqrt' | 'log';
 }
 
 /**
@@ -51,7 +56,8 @@ export const getByteSurprisalHistogramConfig = (): HistogramBaseConfig => ({
 export const getDeltaByteSurprisalHistogramConfig = (): HistogramBaseConfig => ({
   label: tr("Δinformation per byte histogram"),
   no_bins: 20,
-  xAxisTickSkip: 1,  // x轴刻度数字绘制间隔
+  xAxisTickSkip: 1,
+  xAxisTickRound: true,
   extent: [-5, 5],
   averageLabel: tr("Δ bits/Byte"),
   showLeftInfinity: true,
@@ -68,14 +74,14 @@ export const getSurprisalProgressConfig = (): ScatterPlotBaseConfig => ({
 });
 
 /**
- * 获取 Semantic score 直方图配置（基于 match_score，归一化 0-1）
+ * 获取 Raw score normed 直方图配置（归一化 0-1）
  */
-export const getSemanticScoreHistogramConfig = (): HistogramBaseConfig => ({
-  label: tr("semantic score histogram"),
+export const getRawScoreNormedHistogramConfig = (): HistogramBaseConfig => ({
+  label: tr("semantic raw score histogram"),
   no_bins: 20,
   xAxisTickSkip: 1,
+  xAxisTickRound: true,
   extent: [0, 1],
-  averageLabel: tr("score"),
   yScaleType: 'sqrt',
 });
 

client/src/ts/utils/visualizationUpdater.ts

@@ -23,18 +23,38 @@ import {
 } from './dataValidation';
 import {
     calculateTextStats,
+    computeP90,
     type TextStats
 } from './textStatistics';
 import {
     getTokenSurprisalHistogramConfig,
     getSurprisalProgressConfig,
-    getSemanticScoreHistogramConfig
+    getRawScoreNormedHistogramConfig
 } from "./visualizationConfigs";
 import { isFiniteNumber } from './Util';
-import { getSemanticSimilarityColor } from './SurprisalColorConfig';
+import { getSemanticSimilarityColor, HISTOGRAM_MIN_ALPHA } from './SurprisalColorConfig';
 import { showAlertDialog } from '../ui/dialog';
 import { tr } from '../lang/i18n-lite';
+import { computeExpectedCounts } from './lognormalFit';
+import { findSignalThreshold, type SignalThresholdBin } from './signalThresholdDetector';
 import { getSemanticAnalysisEnabled } from './semanticAnalysisManager';
+import { renderTopkChartFullHtml } from './topkChartUtils';
+
+
+/**
+ * P(signal | raw_score_normed = s) 复用 findSignalThreshold 的 bins
+ * 每个样本 s 落入对应 bin，P(signal) = (obsInBin - expInBin) / obsInBin
+ */
+function signalProbFromBins(scores: number[], bins: SignalThresholdBin[]): number[] {
+    if (scores.length === 0 || bins.length === 0) return [];
+    const tauLefts = bins.map((b) => b.tauLeft);
+    return scores.map((s) => {
+        const i = Math.max(0, Math.min(bins.length - 1, d3.bisectRight(tauLefts, s) - 1));
+        const b = bins[i]!;
+        if (s < b.tauLeft || s >= b.tauRight) return 0;
+        return b.obsInBin > 0 ? Math.max(0, Math.min(1, (b.obsInBin - b.expInBin) / b.obsInBin)) : 0;
+    });
+}
 
 /**
  * 可视化更新依赖
@@ -44,7 +64,7 @@ export interface VisualizationDependencies {
     highlightController: HighlightController;
     textInputController: TextInputController;
     stats_frac: Histogram;
-    stats_semantic_score: Histogram;
+    stats_raw_score_normed: Histogram;
     stats_surprisal_progress: ScatterPlot;
     appStateManager: AppStateManager;
     surprisalColorScale: d3.ScaleSequential<string>;
@@ -140,7 +160,7 @@ export class VisualizationUpdater {
     /**
      * 计算展示结果：仅信息密度 / 仅语义 / 联合（两者一致时）
      */
-    private computeDisplayResult(): (FrontendAnalyzeResult & { attentionScores?: number[]; attentionRawScores?: number[] }) | null {
+    private computeDisplayResult(): (FrontendAnalyzeResult & { rawScoresNormed?: number[]; attentionRawScores?: number[] }) | null {
         const info = this.currentState.infoDensityData;
         const sem = this.currentState.semanticData;
         const infoResult = info?.result as FrontendAnalyzeResult | undefined;
@@ -175,7 +195,7 @@ export class VisualizationUpdater {
                     mergedTokens: filteredMerged,
                     bpe_strings: filteredMerged,
                     originalToMergedMap: filteredOriginalToMergedMap,
-                    attentionScores: this.normalizeScoresForColor(scores),
+                    rawScoresNormed: this.normalizeScoresForColor(scores),
                     attentionRawScores: scores,
                 };
             }
@@ -194,7 +214,7 @@ export class VisualizationUpdater {
         const trimmed = text.trim();
         const tokenHistogramItem = document.getElementById('token_histogram_item');
         const surprisalProgressItem = document.getElementById('surprisal_progress_item');
-        const semanticScoreItem = document.getElementById('semantic_score_histogram_item');
+        const rawScoreNormedItem = document.getElementById('raw_score_normed_histogram_item');
 
         const infoText = (this.currentState.infoDensityData?.request?.text ?? '').trim();
         const semText = (this.currentState.semanticData?.text ?? '').trim();
@@ -212,7 +232,7 @@ export class VisualizationUpdater {
 
         if (tokenHistogramItem) tokenHistogramItem.style.display = showInfoDensity ? '' : 'none';
         if (surprisalProgressItem) surprisalProgressItem.style.display = showInfoDensity ? '' : 'none';
-        if (semanticScoreItem) semanticScoreItem.style.display = showSemantic ? '' : 'none';
+        if (rawScoreNormedItem) rawScoreNormedItem.style.display = showSemantic ? '' : 'none';
 
         // pending 时渲染空统计图（坐标轴 + 空柱体/散点），避免空白
         if (showInfoDensity && mode === 'infoDensity') {
@@ -226,24 +246,25 @@ export class VisualizationUpdater {
             if (progressTitle && progressConfig.label) progressTitle.textContent = progressConfig.label;
         }
         if (showSemantic && mode === 'semantic') {
-            const semanticConfig = getSemanticScoreHistogramConfig();
-            this.deps.stats_semantic_score.update({ ...semanticConfig, data: [], colorScale: () => 'transparent' });
-            const titleEl = document.getElementById('semantic_score_histogram_title');
-            if (titleEl) titleEl.textContent = semanticConfig.label;
+            const rawScoreNormedConfig = getRawScoreNormedHistogramConfig();
+            this.deps.stats_raw_score_normed.update({ ...rawScoreNormedConfig, data: [], colorScale: () => 'transparent' });
+            const titleEl = document.getElementById('raw_score_normed_histogram_title');
+            if (titleEl) titleEl.textContent = rawScoreNormedConfig.label;
         }
     }
 
     /**
      * 重新渲染直方图（内部方法）
-     * 仅信息密度：只显示 token/surprisal progress；仅语义：只显示 semantic score；联合：全部显示
+     * 仅信息密度：只显示 token/surprisal progress；仅语义：只显示 raw score normed；联合：全部显示
+     * @param skipLmfUpdate 为 true 时跳过 lmf.update（主题切换时由 rerenderOnThemeChange 统一重绘，避免竞态）
      */
-    private rerenderHistogramsInternal(): void {
+    private rerenderHistogramsInternal(skipLmfUpdate = false): void {
         const hasInfoDensity = !!this.currentState.infoDensityData;
         const displayResult = this.computeDisplayResult();
 
         const tokenHistogramItem = document.getElementById('token_histogram_item');
         const surprisalProgressItem = document.getElementById('surprisal_progress_item');
-        const semanticScoreItem = document.getElementById('semantic_score_histogram_item');
+        const rawScoreNormedItem = document.getElementById('raw_score_normed_histogram_item');
 
         if (hasInfoDensity) {
             const currentSurprisals = this.currentState.currentSurprisals;
@@ -257,6 +278,7 @@ export class VisualizationUpdater {
                     colorScale: this.deps.surprisalColorScale,
                     averageValue: currentTokenAvg ?? undefined,
                     p90Value: currentTokenP90 ?? undefined,
+                    p90Label: tokenHistogramConfig.averageLabel,
                 });
                 const titleElement = document.getElementById('token_histogram_title');
                 if (titleElement) titleElement.textContent = tokenHistogramConfig.label;
@@ -279,31 +301,125 @@ export class VisualizationUpdater {
             if (surprisalProgressItem) surprisalProgressItem.style.display = 'none';
         }
 
-        const matchScores = displayResult?.attentionScores;
-        const validMatchScores = matchScores?.filter((s) => typeof s === 'number' && isFinite(s));
-        if (validMatchScores && validMatchScores.length > 0) {
-            const semanticConfig = getSemanticScoreHistogramConfig();
-            const avgScore = validMatchScores.reduce((a, b) => a + b, 0) / validMatchScores.length;
-            const colorScale = (v: number) => getSemanticSimilarityColor(v);
-            this.deps.stats_semantic_score.update({
-                ...semanticConfig,
-                data: validMatchScores,
+        const rawScoresNormed = displayResult?.rawScoresNormed;
+        const validRawScoresNormed = rawScoresNormed?.filter((s) => typeof s === 'number' && isFinite(s));
+        if (validRawScoresNormed && validRawScoresNormed.length > 0) {
+            const rawScoreNormedConfig = getRawScoreNormedHistogramConfig();
+            const colorScale = (v: number) => getSemanticSimilarityColor(v, HISTOGRAM_MIN_ALPHA);
+            // 默认关闭 verbose；浏览器控制台执行 window.signalThresholdVerbose = true 后重新搜索可开启
+            const verbose = !!(window as Window & { signalThresholdVerbose?: boolean }).signalThresholdVerbose;
+            const signalThresholdResult = findSignalThreshold(validRawScoresNormed, verbose);
+            console.log('[signalThreshold] 最终结果:', signalThresholdResult !== null
+                ? (() => {
+                    const t = signalThresholdResult!.threshold;
+                    const below = validRawScoresNormed.filter((s) => s < t).length;
+                    const quantile = validRawScoresNormed.length > 0 ? (below / validRawScoresNormed.length) : 0;
+                    return `threshold=${t.toFixed(4)} confidence=${signalThresholdResult!.confidence.toFixed(2)} quantile=${quantile.toFixed(4)} (${below}/${validRawScoresNormed.length} below)`;
+                })()
+                : 'null（无信号）');
+            if (!verbose) {
+                console.log('[signalThreshold] 提示：控制台执行 window.signalThresholdVerbose = true 后重新搜索可查看完整 bin 扫描日志');
+            }
+            const fitResult = validRawScoresNormed.length >= 2 && signalThresholdResult != null
+                ? {
+                    mu: signalThresholdResult.mu,
+                    sigma: signalThresholdResult.sigma,
+                    expectedCounts: computeExpectedCounts(
+                        signalThresholdResult.mu,
+                        signalThresholdResult.sigma,
+                        rawScoreNormedConfig.extent as [number, number],
+                        rawScoreNormedConfig.no_bins,
+                        validRawScoresNormed.length
+                    ),
+                }
+                : null;
+            console.log('[raw score normed histogram] fitted log-normal μ, σ:', fitResult ? [fitResult.mu, fitResult.sigma] : 'failed');
+            const signalProbs = signalThresholdResult != null
+                ? signalProbFromBins(validRawScoresNormed, signalThresholdResult.bins)
+                : [];
+            /**
+             * P_pw：后验信号概率的简化映射，x <= threshold 时为 0，x > threshold 时为 1
+             * pw_score = score × P_pw
+             * 基于 rawScoresNormed 全数组计算，保证与 token 索引对齐
+             */
+            const rawScoresNormedFull = displayResult!.rawScoresNormed ?? [];
+            const t = signalThresholdResult?.threshold ?? 0;
+            const pPwValues = signalThresholdResult != null
+                ? rawScoresNormedFull.map((s) => (typeof s === 'number' && isFinite(s) && s > t ? 1 : 0))
+                : [];
+            const pwScores = signalThresholdResult != null
+                ? rawScoresNormedFull.map((s) => (typeof s === 'number' && isFinite(s) && s > t ? s : 0))
+                : [];
+            const probCurveData = signalProbs.length > 0
+                ? (() => {
+                    const pairs = validRawScoresNormed.map((x, i) => ({ x, y: signalProbs[i]! })).sort((a, b) => a.x - b.x);
+                    return { x: pairs.map(p => p.x), y: pairs.map(p => p.y) };
+                })()
+                : undefined;
+            const signalThresholdPercentile = signalThresholdResult != null && validRawScoresNormed.length > 0
+                ? Math.round((validRawScoresNormed.filter((s) => s < signalThresholdResult.threshold).length / validRawScoresNormed.length) * 100)
+                : undefined;
+            this.deps.stats_raw_score_normed.update({
+                ...rawScoreNormedConfig,
+                data: validRawScoresNormed,
                 colorScale,
-                averageValue: avgScore,
+                fitExpectedCounts: fitResult?.expectedCounts,
+                showProbCurve: true,
+                probCurveData: probCurveData?.x.length ? probCurveData : undefined,
+                signalThreshold: signalThresholdResult?.threshold ?? undefined,
+                signalThresholdPercentile: signalThresholdPercentile ?? undefined,
             });
-            const titleEl = document.getElementById('semantic_score_histogram_title');
-            if (titleEl) titleEl.textContent = semanticConfig.label;
-            if (semanticScoreItem) semanticScoreItem.style.display = '';
-        } else if (semanticScoreItem) {
-            semanticScoreItem.style.display = 'none';
+            const titleEl = document.getElementById('raw_score_normed_histogram_title');
+            if (titleEl) titleEl.textContent = rawScoreNormedConfig.label;
+            if (rawScoreNormedItem) rawScoreNormedItem.style.display = '';
+
+            const colorSourceEl = document.getElementById('semantic_color_source_select') as HTMLSelectElement | null;
+            const scoresForColor = colorSourceEl?.value === 'signal_probability' ? pPwValues
+                : colorSourceEl?.value === 'pw_score' ? pwScores
+                : (displayResult!.rawScoresNormed ?? []);
+
+            if (fitResult != null) {
+                const resultWithExt = { ...displayResult, signalProbs, pPwValues, pwScores };
+                this.deps.highlightController.updateCurrentData({ result: resultWithExt, signalProbs, pPwValues, pwScores });
+                if (!skipLmfUpdate) {
+                    this.deps.lmf.update({ ...resultWithExt, pwScores, colorScores: scoresForColor } as FrontendAnalyzeResult & { pPwValues?: number[]; pwScores?: number[]; colorScores?: number[] });
+                }
+            } else {
+                this.deps.highlightController.updateCurrentData({ result: displayResult });
+                if (!skipLmfUpdate) {
+                    this.deps.lmf.update({ ...displayResult, colorScores: scoresForColor } as FrontendAnalyzeResult & { colorScores?: number[] });
+                }
+            }
+        } else {
+            if (rawScoreNormedItem) rawScoreNormedItem.style.display = 'none';
+            if (displayResult) this.deps.highlightController.updateCurrentData({ result: displayResult });
         }
     }
 
-    /**
-     * 重新渲染直方图（公开方法，供外部调用，如主题切换时）
-     */
+    /** 重新渲染直方图（供外部调用） */
     public rerenderHistograms(): void {
-        this.rerenderHistogramsInternal();
+        this.rerenderHistogramsInternal(false);
+    }
+
+    /** 仅更新语义着色源（color source 切换时调用，不重新拟合） */
+    public updateSemanticColorSource(): void {
+        const cd = this.deps.highlightController.getCurrentData();
+        const r = cd?.result as (FrontendAnalyzeResult & { rawScoresNormed?: number[] }) | undefined;
+        if (!r?.rawScoresNormed?.length) return;
+        const el = document.getElementById('semantic_color_source_select') as HTMLSelectElement | null;
+        const v = el?.value;
+        const scoresForColor = v === 'signal_probability' ? (cd!.pPwValues ?? [])
+            : v === 'pw_score' ? (cd!.pwScores ?? [])
+            : r.rawScoresNormed;
+        this.deps.lmf.update({ ...r, pPwValues: cd!.pPwValues, pwScores: cd!.pwScores, colorScores: scoresForColor } as FrontendAnalyzeResult & { pPwValues?: number[]; pwScores?: number[]; colorScores?: number[] });
+    }
+
+    /** 主题切换时调用：在样式生效后统一重绘直方图与文本（rgba 透出背景，需等新主题生效） */
+    public rerenderOnThemeChange(): void {
+        requestAnimationFrame(() => requestAnimationFrame(() => {
+            this.rerenderHistogramsInternal(true);
+            this.deps.lmf.reRenderCurrent();
+        }));
     }
 
     /**
@@ -486,8 +602,7 @@ export class VisualizationUpdater {
         res: {
             model?: string;
             token_attention?: Array<{ offset: [number, number]; raw: string; score: number }>;
-            debug_abbrev?: string;
-            debug_top10?: Array<{ token: string; prob: number }>;
+            debug_info?: { abbrev?: string; topk_tokens?: string[]; topk_probs?: number[] };
         },
         text?: string
     ): void {
@@ -523,12 +638,16 @@ export class VisualizationUpdater {
             enableRenderAnimation: false,
             semanticAnalysisMode: getSemanticAnalysisEnabled(),
         }, false);
-        this.deps.lmf.update(displayResult);
         this.clearHighlights();
+        // 仅由 rerenderHistogramsInternal 调用 lmf.update，避免与 handleSemanticResponse 的重复调用导致语义渲染双重叠加
         this.rerenderHistogramsInternal();
         this.syncSemanticUiFromConfig();
 
-        this.updateSemanticDebugInfo(res.debug_abbrev, res.debug_top10);
+        const di = res.debug_info;
+        const top10 = (di?.topk_tokens && di?.topk_probs)
+            ? di.topk_tokens.map((token, i) => ({ token, prob: di.topk_probs![i] ?? 0 }))
+            : undefined;
+        this.updateSemanticDebugInfo(di?.abbrev, top10);
     }
 
     /** 更新文本渲染区下方的 debug 信息（abbrev + top10） */
@@ -553,6 +672,7 @@ export class VisualizationUpdater {
         if (top10?.length) {
             const items = top10.map((t) => `'${esc(t.token)}(${(t.prob * 100).toFixed(1)}%)'`);
             parts.push(`<div class="semantic-debug-top10">[${items.join(', ')}]</div>`);
+            parts.push(renderTopkChartFullHtml(top10));
         }
         el.innerHTML = parts.join('');
     }
@@ -561,7 +681,7 @@ export class VisualizationUpdater {
         res: { model?: string },
         tokenAttention: Array<{ offset: [number, number]; raw: string; score: number }>,
         text: string
-    ): (FrontendAnalyzeResult & { attentionScores: number[]; attentionRawScores: number[] }) | null {
+    ): (FrontendAnalyzeResult & { rawScoresNormed: number[]; attentionRawScores: number[] }) | null {
         const safeText = text.trim();
         if (!safeText) return null;
         const syntheticTokens = tokenAttention.map((t) => ({
@@ -583,7 +703,7 @@ export class VisualizationUpdater {
             mergedTokens,
             originalToMergedMap,
             originalText: safeText,
-            attentionScores: this.normalizeScoresForColor(scores),
+            rawScoresNormed: this.normalizeScoresForColor(scores),
             attentionRawScores: scores,
         };
     }
@@ -637,7 +757,7 @@ export class VisualizationUpdater {
     }
 
     /**
-     * 将 score 归一化到 [0,1] 用于染色（0-max 归一化：norm = score / max）
+     * 将 raw score 归一化到 [0,1] 用于染色（0-max 归一化：norm = raw_score / max）
      * NaN/Inf 不参与 max 计算，映射为 0
      */
     private normalizeScoresForColor(scores: number[]): number[] {

client/src/ts/vis/GLTR_Text_Box.ts

@@ -43,12 +43,15 @@ export enum GLTR_Mode {
     fract_p
 }
 
-/** tokenData：信息密度模式为 FrontendToken，Semantic analysis 模式下附加 matchScore */
-export type TokenDataForRender = FrontendToken & { matchScore?: number };
+/** tokenData：信息密度模式为 FrontendToken，Semantic analysis 模式下附加 rawScoreNormed */
+export type TokenDataForRender = FrontendToken & { rawScoreNormed?: number };
 
 /** 语义模式下的 Tooltip 展示字段 */
 export type SemanticRenderFields = {
-    matchScore?: number;
+    pwScore?: number;
+    /** 信号概率 P_pw：x<=threshold 为 0，x>threshold 为 1 */
+    signalProb?: number;
+    rawScoreNormed?: number;
     /** Attention 分析时的原始 score（未归一化） */
     rawScore?: number;
 };
@@ -62,9 +65,9 @@ export type GLTR_HoverEvent = { hovered: boolean, d: GLTR_RenderItem, event?: Mo
 
 /** 从 token 中安全提取语义展示字段，无需类型断言 */
 function extractSemanticFields(token: TokenDataForRender): SemanticRenderFields | undefined {
-    const matchScore = "matchScore" in token && typeof token.matchScore === "number" ? token.matchScore : undefined;
-    if (matchScore === undefined) return undefined;
-    return { matchScore };
+    const rawScoreNormed = "rawScoreNormed" in token && typeof token.rawScoreNormed === "number" ? token.rawScoreNormed : undefined;
+    if (rawScoreNormed === undefined) return undefined;
+    return { rawScoreNormed };
 }
 
 export class GLTR_Text_Box extends VComponent<FrontendAnalyzeResult> {
@@ -87,7 +90,7 @@ export class GLTR_Text_Box extends VComponent<FrontendAnalyzeResult> {
         // Minimap 配置
         enableMinimap: false,  // 是否启用 minimap（默认关闭）
         minimapWidth: getMinimapWidthFromCSS(),  // minimap 宽度（像素），从 CSS 变量读取
-        // Semantic analysis 模式：为 true 时按 attention score 染色
+        // Semantic analysis 模式：为 true 时按 raw score normed 染色
         semanticAnalysisMode: false,
     };
     
@@ -237,9 +240,10 @@ export class GLTR_Text_Box extends VComponent<FrontendAnalyzeResult> {
             this.positionCalculator = new TokenPositionCalculator(baseNode);
         }
 
-        const rdExt = rd as FrontendAnalyzeResult & { attentionScores?: number[] };
-        const attentionScores = rdExt.attentionScores;
-        const isSemantic = this.options.semanticAnalysisMode && attentionScores?.length;
+        const rdExt = rd as FrontendAnalyzeResult & { rawScoresNormed?: number[]; colorScores?: number[] };
+        const rawScoresNormed = rdExt.rawScoresNormed;
+        const colorScores = (rdExt.colorScores?.length ? rdExt.colorScores : undefined) ?? rawScoresNormed;
+        const isSemantic = this.options.semanticAnalysisMode && colorScores?.length;
         
         // Semantic analysis 模式：按 BPE（merged tokens）；否则按 BPE
         const rdForPositions: FrontendAnalyzeResult = rd;
@@ -262,7 +266,7 @@ export class GLTR_Text_Box extends VComponent<FrontendAnalyzeResult> {
                     charToByteIndexMap: this._current.charToByteIndexMap,
                 }
                 : undefined,
-            semantic: isSemantic ? { analysisMode: true, matchScores: attentionScores } : undefined,
+            semantic: isSemantic ? { analysisMode: true, rawScoresNormed: colorScores } : undefined,
         };
         this.svgOverlayManager = new SvgOverlayManager(baseNode, overlayOptions);
 
@@ -616,30 +620,16 @@ export class GLTR_Text_Box extends VComponent<FrontendAnalyzeResult> {
     
     /**
      * 设置主题变化监听器
+     * 仅更新 fracScale/diffScale；重渲染由 initThemeManager 的 onThemeChange -> rerenderOnThemeChange 统一触发
      */
     private setupThemeListener(): void {
-        // 使用MutationObserver监听data-theme属性的变化
         const observer = new MutationObserver((mutations) => {
             mutations.forEach((mutation) => {
                 if (mutation.type === 'attributes' && mutation.attributeName === 'data-theme') {
-                    // 主题变化时，更新颜色scale并重新渲染
                     this.updateColorScales();
-                    if (this.currentRenderData) {
-                        // 主题切换时禁用动画，立即重新渲染
-                        const originalAnimationSetting = this.options.enableRenderAnimation;
-                        this.options.enableRenderAnimation = false;
-                        // 重新渲染当前数据
-                        this._render(this.currentRenderData);
-                        // 恢复动画设置
-                        setTimeout(() => {
-                            this.options.enableRenderAnimation = originalAnimationSetting;
-                        }, 100);
-                    }
                 }
             });
         });
-        
-        // 开始观察document.documentElement的data-theme属性
         observer.observe(document.documentElement, {
             attributes: true,
             attributeFilter: ['data-theme']
@@ -662,18 +652,23 @@ export class GLTR_Text_Box extends VComponent<FrontendAnalyzeResult> {
      */
     private addTokenEventListeners(element: SVGGElement, tokenIndex: number, rd: FrontendAnalyzeResult): void {
         const rdExt = rd as FrontendAnalyzeResult & {
-            attentionScores?: number[];
+            rawScoresNormed?: number[];
             attentionRawScores?: number[];
+            pPwValues?: number[];
+            pwScores?: number[];
         };
-        const hasAttentionScores = rdExt.attentionScores?.length && tokenIndex < rdExt.attentionScores.length;
-        const showTooltip = true;  // 始终显示 tooltip，semantic 部分在 hasAttentionScores 时填充
+        const hasRawScoresNormed = rdExt.rawScoresNormed?.length && tokenIndex < rdExt.rawScoresNormed.length;
+        const showTooltip = true;  // 始终显示 tooltip，semantic 部分在 hasRawScoresNormed 时填充
 
         const tokenData = rd.bpe_strings[tokenIndex] as TokenDataForRender;
         let semantic = showTooltip ? extractSemanticFields(tokenData) : undefined;
-        if (showTooltip && hasAttentionScores && rdExt.attentionScores) {
-            const attnScore = rdExt.attentionScores[tokenIndex];
+        if (showTooltip && hasRawScoresNormed && rdExt.rawScoresNormed) {
+            // rawScoreNormed 始终用 rawScoresNormed，与 color source 无关
+            const attnScore = rdExt.rawScoresNormed[tokenIndex];
             const rawScore = rdExt.attentionRawScores?.[tokenIndex];
-            semantic = { ...semantic, matchScore: attnScore, rawScore } as SemanticRenderFields;
+            const signalProb = rdExt.pPwValues?.[tokenIndex];  // P_pw：x<=threshold 为 0，x>threshold 为 1
+            const pwScore = rdExt.pwScores?.[tokenIndex];
+            semantic = { ...semantic, rawScoreNormed: attnScore, rawScore, signalProb, pwScore } as SemanticRenderFields;
         }
 
         const handleMouseEnter = (event: MouseEvent) => {

client/src/ts/vis/Histogram.ts

@@ -1,16 +1,46 @@
 import { VComponent } from "./VisComponent";
 import { D3Sel } from "../utils/Util";
 import { SimpleEventHandler } from "../utils/SimpleEventHandler";
+import { tr } from "../lang/i18n-lite";
 import * as d3 from "d3";
 import { schemeDark2 } from "d3";
 
 const averageNumberFormat = d3.format('.2f');
 
+/** 1-2-5 decade 模式生成非线性 y 轴刻度，最多 maxTicks 个 */
+function getNonLinearTickValues(maxCount: number, maxTicks = 10): number[] {
+    if (maxCount <= 0) return [0];
+    const ticks: number[] = [0];
+    const base = [1, 2, 5];
+    let decade = 1;
+    while (decade <= maxCount) {
+        for (const b of base) {
+            const v = b * decade;
+            if (v <= maxCount) ticks.push(v);
+        }
+        decade *= 10;
+    }
+    if (ticks[ticks.length - 1] !== maxCount) ticks.push(maxCount);
+    if (ticks.length <= maxTicks) return ticks;
+    const result: number[] = [];
+    for (let i = 0; i < maxTicks; i++) {
+        const idx = Math.round((i / (maxTicks - 1)) * (ticks.length - 1));
+        result.push(ticks[idx]);
+    }
+    return [...new Set(result)].sort((a, b) => a - b);
+}
+
+/** 单边：固定值或 'auto'（从 data 解析） */
+export type HistogramExtentBound = number | 'auto';
+
+/** extent：'auto' 等价于 ['auto','auto']，支持双边独立配置 */
+export type HistogramExtent = [HistogramExtentBound, HistogramExtentBound] | 'auto';
+
 export type HistogramData = {
     data: number[],
     label?: string,
     no_bins: number,
-    extent: number[],
+    extent: HistogramExtent,
     colorScale: (value: number) => string,  // 添加颜色 scale
     averageValue?: number,
     p90Value?: number,
@@ -18,8 +48,21 @@ export type HistogramData = {
     p90Label?: string,
     showLeftInfinity?: boolean,
     showRightInfinity?: boolean,
-    xAxisTickSkip?: number,  // x轴刻度数字绘制间隔，0表示不跳过，1表示隔一个绘制一个（0,2,4...）
-    yScaleType?: 'linear' | 'sqrt'  // y轴尺度：linear 线性，sqrt 平方根（压缩高值、放大低值，适合分布悬殊的直方图）
+    /** x轴刻度数字绘制间隔，0表示不跳过，1表示隔一个绘制一个（0,2,4...） */
+    xAxisTickSkip?: number,
+    /** x轴刻度凑整：true=仅显示 step 整数倍处的标签（与 tickSkip 配合），false/undefined=显示全部 */
+    xAxisTickRound?: boolean;
+    yScaleType?: 'linear' | 'sqrt' | 'log'  // y轴尺度：linear 线性，sqrt 平方根，log 对数（指数刻度，从 0 开始）
+    /** 拟合分布的每个 bin 期望计数，用于绘制横虚线标识（如指数噪声拟合） */
+    fitExpectedCounts?: number[];
+    /** 是否叠加 prob 曲线（共用 x 轴，左侧新 y 轴 0–1） */
+    showProbCurve?: boolean;
+    /** 曲线数据：x=raw_score_normed，y=prob（0–1），P(signal) 按 findSignalThreshold 的 bin 分块估计，(obs-exp)/obs */
+    probCurveData?: { x: number[]; y: number[] };
+    /** 信号阈值竖线：归一化分数，用于 raw_score_normed 直方图 */
+    signalThreshold?: number | null;
+    /** 信号阈值对应的分位数（0–100），用于 label 显示 τ = pXX */
+    signalThresholdPercentile?: number | null;
 }
 
 
@@ -68,6 +111,9 @@ export class Histogram extends VComponent<HistogramData> {
             .attr('class', 'y-axis')
             .attr('transform', `translate(${op.width - 33},0)`)
 
+        this.layers.bg.append('g')
+            .attr('class', 'y-axis-prob')
+
         // 背景面板：避免柱体与整体页面纯白背景混淆
         this.layers.bg.insert('rect', ':first-child')
             .attr('class', 'panel-bg')
@@ -88,8 +134,18 @@ export class Histogram extends VComponent<HistogramData> {
     protected _render(rD: HistogramData): void {
         const op = this.options;
 
-        // extent 是必选参数，直接使用
-        const extent = rD.extent;
+        // extent 解析：'auto' 等价于 ['auto','auto']，支持双边独立配置
+        const [loSpec, hiSpec]: [HistogramExtentBound, HistogramExtentBound] =
+            rD.extent === 'auto' ? ['auto', 'auto'] : rD.extent;
+        const finite = rD.data.filter((d) => typeof d === 'number' && isFinite(d));
+        const [dataLo, dataHi] = finite.length > 0
+            ? (d3.extent(finite) as [number, number])
+            : [0, 1];
+        const fallbackLo = finite.length <= 1 ? dataLo - 0.5 : dataLo;
+        const fallbackHi = finite.length <= 1 ? dataHi + 0.5 : dataHi;
+        const lo = loSpec === 'auto' ? fallbackLo : loSpec;
+        const hi = hiSpec === 'auto' ? fallbackHi : hiSpec;
+        const extent: [number, number] = lo > hi ? [lo, lo] : [lo, hi];
 
         // 计算bin宽度
         const binWidth = (extent[1] - extent[0]) / rD.no_bins;
@@ -111,7 +167,7 @@ export class Histogram extends VComponent<HistogramData> {
         // 如果指定了 extent，确保使用 extent 作为 domain，而不是 nice() 调整后的 domain
         // 这样可以保证 extent 的上限被正确使用，即使数据被截断了
         // 使用 extent 作为 domain，确保范围正确
-        const padding = { left: 10, right: 35 };
+        const padding = { left: rD.showProbCurve ? 35 : 10, right: 35 };
         let valueScale = d3.scaleLinear().domain([extent[0], extent[1]]).range([padding.left, op.width - padding.right]);
 
         const hasAverageValue = typeof rD.averageValue === 'number' && Number.isFinite(rD.averageValue);
@@ -146,15 +202,25 @@ export class Histogram extends VComponent<HistogramData> {
             console.warn('Invalid maxCount for histogram:', maxCount);
             maxCount = 1;
         }
+        if (rD.fitExpectedCounts && rD.fitExpectedCounts.length > 0) {
+            const fitMax = d3.max(rD.fitExpectedCounts) ?? 0;
+            if (isFinite(fitMax) && fitMax > maxCount) maxCount = fitMax;
+        }
 
         const useSqrt = rD.yScaleType === 'sqrt';
-        const countScale = useSqrt
-            ? d3.scaleSqrt().domain([0, maxCount]).range([op.height - op.margin_bottom, op.margin_top])
-            : d3.scaleLinear().domain([0, maxCount]).nice().range([op.height - op.margin_bottom, op.margin_top]);
-
-        const adjustWidth = (bandH: number) => {
-            if (!isFinite(bandH) || bandH <= 0) return 0;
-            return (bandH > 5) ? Math.max(0, bandH - 2) : (0.8 * bandH);
+        const useLog = rD.yScaleType === 'log';
+        const countScale = useLog
+            ? d3.scaleSymlog().domain([0, Math.max(1, maxCount)]).range([op.height - op.margin_bottom, op.margin_top])
+            : useSqrt
+                ? d3.scaleSqrt().domain([0, maxCount]).range([op.height - op.margin_bottom, op.margin_top])
+                : d3.scaleLinear().domain([0, maxCount]).nice().range([op.height - op.margin_bottom, op.margin_top]);
+
+        // 与 d3 scaleBand 一致：bandwidth = step * (1 - paddingInner)，gap = step * paddingInner
+        // no_bins=20 时 barWidth:gap ≈ 2.875:1 → paddingInner ≈ 0.258
+        const PADDING_INNER = 0.15;
+        const adjustWidth = (step: number) => {
+            if (!isFinite(step) || step <= 0) return 0;
+            return step * (1 - PADDING_INNER);
         };
 
         const getBandWidth = (d: d3.Bin<number, number>) => valueScale(d.x1) - valueScale(d.x0);
@@ -187,9 +253,7 @@ export class Histogram extends VComponent<HistogramData> {
                     return isFinite(w) && w > 0 ? w : 1;
                 },
                 height: d => {
-                    if (d.length === 0) {
-                        return 0;
-                    }
+                    if (d.length === 0) return 0;
                     const h = op.height - op.margin_bottom - countScale(d.length);
                     return isFinite(h) && h > 0 ? h : 1;
                 },
@@ -211,6 +275,29 @@ export class Histogram extends VComponent<HistogramData> {
                 return this._current.selectedBinIndex === i ? 'drop-shadow(0 0 6px rgba(42, 158, 255, 0.8))' : 'none';
             });
 
+        // 拟合分布横虚线：每个 bin 上标识期望计数，宽度与柱体对齐
+        const fitData = rD.fitExpectedCounts && rD.fitExpectedCounts.length === histo.length
+            ? histo.map((d, i) => {
+                const bandWidth = getBandWidth(d);
+                const barWidth = adjustWidth(bandWidth);
+                const base = valueScale(d.x0);
+                const x1 = base + 0.5 * (bandWidth - barWidth);
+                return { x1, x2: x1 + barWidth, y: countScale(Math.max(0, rD.fitExpectedCounts![i])) };
+            })
+            : [];
+        this.layers.main.selectAll('.fit-overlay-line').data(fitData)
+            .join('line')
+            .attr('class', 'fit-overlay-line')
+            .attrs({
+                x1: d => d.x1,
+                x2: d => d.x2,
+                y1: d => d.y,
+                y2: d => d.y,
+            })
+            .style('stroke', 'var(--fit-line-color, #999)')
+            .style('stroke-width', 1)
+            .style('stroke-dasharray', '1,1');
+
         const avgMarkerData = averageX !== null && Number.isFinite(averageX)
             ? [{ x: averageX, value: rD.averageValue as number }]
             : [];
@@ -273,15 +360,50 @@ export class Histogram extends VComponent<HistogramData> {
             .attr('y', op.margin_top)
             .text('p90');
 
+        const hasSignalThreshold = typeof rD.signalThreshold === 'number' && Number.isFinite(rD.signalThreshold);
+        const clampedSignalThreshold = hasSignalThreshold
+            ? Math.min(Math.max(rD.signalThreshold as number, extent[0]), extent[1])
+            : null;
+        const signalThresholdX = hasSignalThreshold && clampedSignalThreshold !== null
+            ? valueScale(clampedSignalThreshold)
+            : null;
+
+        const signalThresholdMarkerData = signalThresholdX !== null && Number.isFinite(signalThresholdX)
+            ? [{ x: signalThresholdX, value: rD.signalThreshold as number, percentile: rD.signalThresholdPercentile }]
+            : [];
+
+        this.layers.fg.selectAll('.signal-threshold-line').data(signalThresholdMarkerData)
+            .join('line')
+            .attr('class', 'signal-threshold-line')
+            .attrs({
+                x1: d => d.x,
+                x2: d => d.x,
+                y1: op.margin_top + 4,
+                y2: op.height - op.margin_bottom
+            })
+            .style('stroke', 'var(--signal-threshold-line-color, #e74c3c)')
+            .style('stroke-width', 1.5)
+            .style('stroke-dasharray', '3,2');
+
+        this.layers.fg.selectAll('.signal-threshold-marker-label').data(signalThresholdMarkerData)
+            .join('text')
+            .attr('class', 'signal-threshold-marker-label sizeLabel')
+            .attr('text-anchor', 'middle')
+            .attr('x', d => d.x)
+            .attr('y', op.margin_top)
+            .text(d => typeof d.percentile === 'number' ? `τ = p${d.percentile}` : 'τ');
+
         const p90LabelData = (typeof rD.p90Value === 'number' && Number.isFinite(rD.p90Value)) ? [rD.p90Value] : [];
+        const p90LabelY = avgLabelData.length > 0 ? Math.max(24, op.margin_top + 10) : Math.max(12, op.margin_top - 2);
         this.layers.fg.selectAll('.p90-label').data(p90LabelData)
             .join('text')
             .attr('class', 'p90-label sizeLabel')
             .attr('text-anchor', 'end')
             .attr('x', op.width * 0.75)
-            .attr('y', Math.max(24, op.margin_top + 10))
+            .attr('y', p90LabelY)
             .text(value => {
-                return `p90 = ${averageNumberFormat(value)} bits`;
+                const suffix = rD.p90Label ? ` ${rD.p90Label}` : '';
+                return `p90 = ${averageNumberFormat(value)}${suffix}`;
             });
 
         const labelData = histo.filter(bin => bin.length > 0);
@@ -408,27 +530,28 @@ export class Histogram extends VComponent<HistogramData> {
             });
 
 
-        const yAxis = d3.axisRight(countScale).tickFormat(op.numberFormat);
-        if (useSqrt) yAxis.ticks(5);  // sqrt 尺度下减少刻度，避免拥挤
+        const yAxis = d3.axisRight(countScale)
+            .tickFormat(useLog ? d3.format('.0f') : op.numberFormat);
+        if (useSqrt || useLog) yAxis.tickValues(getNonLinearTickValues(maxCount, 10));
         this.layers.bg.select('.y-axis').call(<any>yAxis);
         
         const tickValues = [extent[0], ...thresholds, extent[1]];
         const tickSkip = rD.xAxisTickSkip ?? 0;
         
         // Custom tick format: 根据 showLeftInfinity/showRightInfinity 决定是否显示 ±∞
-        // 根据 xAxisTickSkip 参数决定是否显示数字标签（刻度线始终显示）
+        // xAxisTickSkip：减少刻度标签密度；xAxisTickRound：true 时按 step 对齐过滤，false 时按索引跳过
         const xAxisTickFormat = (d: number) => {
-            // 边界值优先处理，避免被 tickSkip 逻辑意外隐藏
-            if (rD.extent) {
-                if (rD.showLeftInfinity && Math.abs(d - rD.extent[0]) < 0.001) return '-∞';
-                if (rD.showRightInfinity && Math.abs(d - rD.extent[1]) < 0.001) return '∞';
-            }
+            if (rD.showLeftInfinity && Math.abs(d - extent[0]) < 0.001) return '-∞';
+            if (rD.showRightInfinity && Math.abs(d - extent[1]) < 0.001) return '∞';
 
-            // 基于值而非索引决定是否显示：检查 d 是否为 step 的整数倍
-            // 这样可以自动对齐到整的刻度，避免起点偏移导致显示非整数刻度
             if (tickSkip > 0) {
-                const step = (tickSkip + 1) * binWidth;
-                if (Math.abs(d / step - Math.round(d / step)) > 1e-9) return '';
+                if (rD.xAxisTickRound) {
+                    const step = (tickSkip + 1) * binWidth;
+                    if (Math.abs(d / step - Math.round(d / step)) > 1e-9) return '';
+                } else {
+                    const idx = tickValues.findIndex((t) => Math.abs(t - d) < 1e-9 * (Math.abs(d) + 1));
+                    if (idx >= 0 && idx % (tickSkip + 1) !== 0) return '';
+                }
             }
 
             return op.numberFormat(d);
@@ -439,6 +562,46 @@ export class Histogram extends VComponent<HistogramData> {
             .tickValues(tickValues);
         this.layers.bg.select('.x-axis').call(<any>xAxis);
 
+        const hasProbCurve = rD.showProbCurve && rD.probCurveData && rD.probCurveData.x.length > 0;
+        if (hasProbCurve) {
+            const probYScale = d3.scaleLinear()
+                .domain([0, 1])
+                .range([op.height - op.margin_bottom, op.margin_top]);
+
+            const probPoints: { x: number; y: number }[] = rD.probCurveData!.x.map((x, i) => ({ x, y: rD.probCurveData!.y[i] ?? 0 }));
+            const probLine = d3.line<{ x: number; y: number }>()
+                .x(d => valueScale(d.x))
+                .y(d => probYScale(d.y))
+                .curve(d3.curveLinear);
+
+            this.layers.fg.selectAll('.prob-curve').data([probPoints])
+                .join('path')
+                .attr('class', 'prob-curve')
+                .attr('d', probLine)
+                .style('fill', 'none')
+                .style('stroke', 'var(--prob-curve-color, rgba(160,200,255,0.85))')
+                .style('stroke-width', 1.5)
+                .style('pointer-events', 'none');
+
+            const probAxis = d3.axisLeft(probYScale)
+                .ticks(5)
+                .tickFormat(d3.format('.1f'));
+            this.layers.bg.select('.y-axis-prob')
+                .attr('transform', `translate(${padding.left},0)`)
+                .call(<any>probAxis);
+
+            this.layers.bg.selectAll('.prob-curve-axis-label').data([1])
+                .join('text')
+                .attr('class', 'prob-curve-axis-label sizeLabel')
+                .attr('text-anchor', 'middle')
+                .attr('transform', `translate(8,${(op.height - op.margin_bottom) / 2 + op.margin_top}) rotate(-90)`)
+                .text(tr('signal ratio'));
+
+        } else {
+            this.layers.fg.selectAll('.prob-curve').remove();
+            this.layers.bg.select('.y-axis-prob').selectAll('*').remove();
+            this.layers.bg.selectAll('.prob-curve-axis-label').remove();
+        }
 
     }
 

client/src/ts/vis/SvgOverlayManager.ts

@@ -18,8 +18,8 @@ export interface DiffOverlayOptions {
 /** 语义分析模式配置 */
 export interface SemanticOverlayOptions {
     analysisMode: boolean;
-    /** 查询匹配时每 token 的匹配度 [0,1] */
-    matchScores?: number[];
+    /** 查询匹配时每 token 的 raw score normed [0,1] */
+    rawScoresNormed?: number[];
 }
 
 export interface SvgOverlayManagerOptions {
@@ -226,7 +226,7 @@ export class SvgOverlayManager {
             group.appendChild(rect);
             // 语义分析模式：在信息密度之上叠加语义高亮（黄色渐变）
             const sem = this.options.semantic;
-            if (sem?.analysisMode && sem.matchScores) {
+            if (sem?.analysisMode && sem.rawScoresNormed) {
                 const overlayRect = this.createSemanticOverlayRect(pos, tokenIndex, rd);
                 group.appendChild(overlayRect);
             }
@@ -326,7 +326,7 @@ export class SvgOverlayManager {
     ): SVGRectElement {
         const rect = document.createElementNS('http://www.w3.org/2000/svg', 'rect');
         const sem = this.options.semantic!;
-        const score = sem.matchScores![tokenIndex];
+        const score = sem.rawScoresNormed![tokenIndex];
         const color = score !== undefined ? getSemanticSimilarityColor(score) : 'transparent';
 
         this.setRectGeometry(rect, pos);

client/src/ts/vis/ToolTip.ts

@@ -1,6 +1,3 @@
-/** Tooltip 显示的 pred_topk 候选数量，与后端 runtime_config.DEFAULT_TOPK 保持一致 */
-const DISPLAY_TOPK = 10;
-
 import { D3Sel, calculateSurprisal, calculateSurprisalDensity } from "../utils/Util";
 import { SimpleEventHandler } from "../utils/SimpleEventHandler";
 import { GLTR_RenderItem } from "./GLTR_Text_Box";
@@ -8,6 +5,8 @@ import type { FrontendToken } from "../api/GLTR_API";
 import * as d3 from "d3";
 import { tr } from "../lang/i18n-lite";
 import { getTokenRenderStyle } from "../utils/tokenRenderStyle";
+import { escapeHtml, visualizeSpecialChars } from "../utils/tokenDisplayUtils";
+import { renderTopkChartHtml } from "../utils/topkChartUtils";
 
 const SEPARATOR = '─────────────';
 
@@ -18,147 +17,6 @@ function renderField(f: DetailField, dc: string, vc: string): string {
     return `<span style="color: ${dc}">${f.label}</span> <span style="color: ${valColor}">${f.value}</span>`;
 }
 
-/**
- * 处理候选词文本，与主token保持一致的处理方式
- * 后端不再处理候选词，直接返回原始解码字符串，前端统一处理
- * @param text 原始文本
- * @returns 处理后的文本（特殊字符可视化 + HTML转义）
- */
-function processCandidateText(text: string): string {
-    // 与主token保持一致：先可视化特殊字符，再HTML转义
-    return escapeHtml(visualizeSpecialChars(text));
-}
-
-/**
- * HTML转义，防止XSS和HTML结构破坏
- * @param text 原始文本
- * @returns 转义后的文本
- */
-function escapeHtml(text: string): string {
-    const div = document.createElement('div');
-    div.textContent = text;
-    return div.innerHTML;
-}
-
-/**
- * 检查字符是否是空白字符
- * @param char 单个字符
- * @returns 是否是空白字符
- */
-function isWhitespaceChar(char: string): boolean {
-    return /\p{White_Space}/u.test(char);
-}
-/**
- * 检查字符是否可打印（常见字符范围）
- * @param char 单个字符
- * @returns 是否可打印
- */
-function isPrintableChar(char: string): boolean {
-
-    // 首先排除所有空白字符
-    if (isWhitespaceChar(char)) {
-        return false;
-    }
-
-    const codePoint = char.codePointAt(0);
-    if (codePoint === undefined) return false;
-    
-    // ASCII 可打印字符范围：32-126（空格到波浪号）
-    if (codePoint >= 32 && codePoint <= 126) {
-        return true;
-    }
-    
-    // 常见 Unicode 范围（中文、日文、韩文、常用符号等）
-    // 基本多文种平面（BMP）中的常见字符范围
-    if (
-        (codePoint >= 0x00A0 && codePoint <= 0x00FF) ||  // 拉丁文补充
-        (codePoint >= 0x0100 && codePoint <= 0x017F) ||  // 拉丁文扩展-A
-        (codePoint >= 0x0180 && codePoint <= 0x024F) ||  // 拉丁文扩展-B
-        (codePoint >= 0x2000 && codePoint <= 0x206F) ||  // 常用标点
-        (codePoint >= 0x2070 && codePoint <= 0x209F) ||  // 上标和下标
-        (codePoint >= 0x20A0 && codePoint <= 0x20CF) ||  // 货币符号
-        (codePoint >= 0x2100 && codePoint <= 0x214F) ||  // 字母式符号
-        (codePoint >= 0x2190 && codePoint <= 0x21FF) ||  // 箭头
-        (codePoint >= 0x2200 && codePoint <= 0x22FF) ||  // 数学运算符
-        (codePoint >= 0x2300 && codePoint <= 0x23FF) ||  // 杂项技术符号
-        (codePoint >= 0x2400 && codePoint <= 0x243F) ||  // 控制图片
-        (codePoint >= 0x2E00 && codePoint <= 0x2E7F) ||  // 补充标点
-        (codePoint >= 0x3000 && codePoint <= 0x303F) ||  // CJK符号和标点
-        (codePoint >= 0x3040 && codePoint <= 0x309F) ||  // 平假名
-        (codePoint >= 0x30A0 && codePoint <= 0x30FF) ||  // 片假名
-        (codePoint >= 0x4E00 && codePoint <= 0x9FFF) ||  // CJK统一汉字
-        (codePoint >= 0xAC00 && codePoint <= 0xD7AF) ||  // 韩文音节
-        (codePoint >= 0xF900 && codePoint <= 0xFAFF) ||  // CJK兼容汉字
-        (codePoint >= 0xFF00 && codePoint <= 0xFFEF)     // 全角字符
-    ) {
-        return true;
-    }
-    
-    return false;
-}
-
-/**
- * 将特殊字符转换为可见的文本表示形式（方案3：文本形式，空格也转义）
- * 对于无法显示的特殊字符，显示其 Unicode 编码
- * @param text 原始文本
- * @returns 转换后的文本，特殊字符已替换为文本标记或 Unicode 编码
- */
-function visualizeSpecialChars(text: string): string {
-    // 先处理常见的特殊字符
-    let result = text
-        .replace(/\r\n/g, '[CRLF]')    // Windows换行 -> [CRLF]
-        .replace(/\n/g, '[LF]')          // 换行 -> [LF]
-        .replace(/\r/g, '[CR]')          // 回车 -> [CR]
-        .replace(/\t/g, '[TAB]')        // Tab -> [TAB]
-        .replace(/\u3000/g, '[FS]')     // 全角空格 -> [FS]
-        .replace(/ /g, '·');            // 空格 -> ·
-        // .replace(/ /g, '␣');            // 空格 -> ␣
-    
-    // 处理其他不可打印或特殊字符，显示 Unicode 编码
-    // 需要跳过已经转换的标记（[...] 内的内容）
-    const processed: string[] = [];
-    let inBracket = false;
-    let bracketContent = '';
-    
-    for (let i = 0; i < result.length; i++) {
-        const char = result[i];
-        
-        if (char === '[') {
-            // 开始标记
-            inBracket = true;
-            bracketContent = '[';
-            processed.push(char);
-        } else if (char === ']' && inBracket) {
-            // 结束标记
-            bracketContent += ']';
-            processed.push(char);
-            inBracket = false;
-            bracketContent = '';
-        } else if (inBracket) {
-            // 在标记内，直接保留
-            bracketContent += char;
-            processed.push(char);
-        } else {
-            // 不在标记内，检查是否可打印
-            if (isPrintableChar(char)) {
-                processed.push(char);
-            } else {
-                // 显示 Unicode 编码
-                const codePoint = char.codePointAt(0);
-                if (codePoint !== undefined) {
-                    const hexCode = codePoint.toString(16).toUpperCase().padStart(4, '0');
-                    processed.push(`[U+${hexCode}]`);
-                } else {
-                    processed.push(char); // 如果无法获取编码，保持原样
-                }
-            }
-        }
-    }
-    
-    return processed.join('');
-}
-
-
 export class ToolTip {
     private predictions: D3Sel;
     private myDetail: D3Sel;
@@ -168,9 +26,6 @@ export class ToolTip {
     private readonly numF = d3.format('.3f');
     private readonly significantF = d3.format('.3g');
     
-    // 缓存：d3 scale（按 maxW 缓存）
-    private scaleCache = new Map<number, d3.ScaleLinear<number, number>>();
-    
     // 缓存：主题颜色
     private themeColors = {
         normalColor: '#333',
@@ -245,16 +100,6 @@ export class ToolTip {
         };
     }
     
-    /**
-     * 获取或创建 scale（带缓存）
-     */
-    private _getScale(maxW: number): d3.ScaleLinear<number, number> {
-        if (!this.scaleCache.has(maxW)) {
-            this.scaleCache.set(maxW, d3.scaleLinear().domain([0, maxW]).range([0, 60]));
-        }
-        return this.scaleCache.get(maxW)!;
-    }
-    
     /**
      * 获取真实的可见视口尺寸和偏移量
      * 优先使用 visualViewport API（解决 iOS Safari 地址栏动态显示/隐藏问题）
@@ -434,15 +279,17 @@ export class ToolTip {
 
         const tokenData = ri.tokenData as FrontendToken;
         const s = ri.semantic;
-        const hasSemantic = s && (s.matchScore !== undefined || s.rawScore !== undefined);
+        const hasSemantic = s && (s.pwScore !== undefined || s.signalProb !== undefined || s.rawScoreNormed !== undefined || s.rawScore !== undefined);
         const hasRealTopk = tokenData?.real_topk != null && Array.isArray(tokenData.real_topk);
         const predTopk = tokenData?.pred_topk ?? [];
         const hasPredictions = predTopk.length > 0;
 
-        // 1. 构建语义区块
+        // 1. 构建语义区块（pw score = raw_score_normed × P_pw，P_pw: x≤threshold 为 0，x>threshold 为 1）
         const semanticRows: string[] = [];
         if (hasSemantic && s) {
-            if (s.matchScore !== undefined) semanticRows.push(renderField({ label: tr('match score:'), value: this.numF(s.matchScore) }, detailColor, valueColor));
+            if (s.pwScore !== undefined) semanticRows.push(renderField({ label: tr('pw score:'), value: this.numF(s.pwScore) }, detailColor, valueColor));
+            if (s.signalProb !== undefined) semanticRows.push(renderField({ label: tr('signal probability:'), value: this.numF(s.signalProb) }, detailColor, valueColor));
+            if (s.rawScoreNormed !== undefined) semanticRows.push(renderField({ label: tr('raw score normed:'), value: this.numF(s.rawScoreNormed) }, detailColor, valueColor));
             if (s.rawScore !== undefined) semanticRows.push(renderField({ label: tr('raw score:'), value: d3.format('.6f')(s.rawScore), valueColor: false }, detailColor, valueColor));
         }
 
@@ -485,20 +332,11 @@ export class ToolTip {
                 .style('display', 'block')
                 .html(() => `<div style="color:${detailColor};padding-left:5px;">${tr('Top-k data not available.')}</div>`);
         } else {
-            const wScale = this._getScale(predTopk[0][1]);
-            this.predictions.selectAll('.row').data(predTopk.slice(0, DISPLAY_TOPK))
-                .join('div')
-                .attr('class', 'row')
-                .style('display', 'table-row')
-                .html(d => {
-                    const color = tokenData.raw != d[0] ? normalColor : selectedColor;
-                    const bar = '<div style="display: table-cell; width:110px;padding-left:5px;">' +
-                        `<div style="display:inline-block;width: ${wScale(d[1])}px;background-color:${color};height: 10px;"></div>` +
-                        ` <div style="display:inline-block;color: ${color};">${this.numF(d[1])}</div>` + "</div>";
-                    const processedText = processCandidateText(d[0]);
-                    const text = `<div style="display: table-cell;color: ${color};padding-right:5px;">${processedText}</div>`;
-                    return `${bar} ${text}`;
-                });
+            const topkData = predTopk.slice(0, 10).map(([token, prob]) => ({ token, prob }));
+            this.predictions.html(renderTopkChartHtml(topkData, {
+                selectedToken: tokenData.raw,
+                numFormat: this.numF,
+            }));
         }
     }
 

math_demo/.streamlit/config.toml

@@ -0,0 +1,2 @@
+[server]
+headless = true

math_demo/requirements.txt

@@ -0,0 +1,4 @@
+streamlit>=1.28.0
+matplotlib>=3.7.0
+numpy>=1.24.0
+scipy>=1.10.0

model_paths.py

@@ -8,12 +8,12 @@ DEFAULT_MODEL = "qwen3.0-0.6b"
 DEFAULT_SEMANTIC_MODEL = "qwen3-0.6b-instruct"
 
 # Semantic analysis 模型（instruct 版本，用于 chat template 与指令理解）
-# 与 qwen3.0-14b 同级：0.6B → 1.7B → 4B → 8B → 14B
 SEMANTIC_MODEL_PATHS = {
     "qwen3-0.6b-instruct": "Qwen/Qwen3-0.6B",
     "qwen3-1.7b-instruct": "Qwen/Qwen3-1.7B",
     # "qwen3-4b-instruct": "Qwen/Qwen3-4B",
     "qwen3-4b-instruct": "Qwen/Qwen3-4B-Instruct-2507",
+    "qwen3.5-0.8b-instruct": "Qwen/Qwen3.5-0.8B",
 }
 
 # 所有可用模型的 HuggingFace 路径映射
@@ -26,6 +26,7 @@ MODEL_PATHS = {
     'qwen3.0-8b': 'Qwen/Qwen3-8B-Base',
     'qwen3.0-14b': 'Qwen/Qwen3-14B-Base',
     'qwen3.0-30b-a3b': 'Qwen/Qwen3-30B-A3B-Base',
+    'qwen3.5-0.8b': 'Qwen/Qwen3.5-0.8B-Base',
     'qwen2.5-32b': 'Qwen/Qwen2.5-32B',
     'qwen2.5-72b': 'Qwen/Qwen2.5-72B',
     

scripts/eval_semantic_submodes.py

@@ -52,7 +52,7 @@ DEFAULT_API_BASE = "http://localhost:5001"
 def analyze_semantic_http(api_base: str, query: str, text: str, submode: str, token: Optional[str] = None, prob_weighted: Optional[bool] = None, timeout: int = 300) -> dict:
     """通过 HTTP 调用 analyze-semantic 接口"""
     url = f"{api_base.rstrip('/')}/api/analyze-semantic"
-    payload: dict = {"query": query, "text": text, "submode": submode}
+    payload: dict = {"query": query, "text": text, "submode": submode, "debug_info": True}
     if prob_weighted is not None:
         payload["prob_weighted"] = prob_weighted
     headers = {"Content-Type": "application/json"}
@@ -87,8 +87,9 @@ def run_eval(api_base: str, submode: str, test_cases: list, token: Optional[str]
             })
             continue
 
-        topk_tokens = res.get("topk_tokens", [])
-        topk_probs = res.get("topk_probs", [])
+        di = res.get("debug_info", {})
+        topk_tokens = di.get("topk_tokens", [])
+        topk_probs = di.get("topk_probs", [])
         token_attention = res.get("token_attention", [])
 
         # 0-max 归一化: score / max ∈ [0, 1]，最大值归一为 1

server.yaml

@@ -503,16 +503,23 @@ paths:
                     score:
                       type: number
                       description: 对 prompt 区域的平均关注度
-              topk_tokens:
-                type: array
-                items:
-                  type: string
-                description: top10 预测 token 列表
-              topk_probs:
-                type: array
-                items:
-                  type: number
-                description: top10 对应的概率
+              debug_info:
+                type: object
+                description: debug_info=true 时返回
+                properties:
+                  abbrev:
+                    type: string
+                    description: 推理原文缩写
+                  topk_tokens:
+                    type: array
+                    items:
+                      type: string
+                    description: top10 预测 token 列表
+                  topk_probs:
+                    type: array
+                    items:
+                      type: number
+                    description: top10 对应的概率
               message:
                 type: string
         400: