| import {VComponent} from "./VisComponent"; |
| import {FrontendAnalyzeResult} from "../api/GLTR_API"; |
| import {D3Sel, calculateSurprisal, calculateSurprisalDensity, buildCharToByteIndexMap} from "../utils/Util"; |
| import {SimpleEventHandler} from "../utils/SimpleEventHandler"; |
| import * as d3 from "d3"; |
| import {RenderAnimator, TokenRenderTask} from "./RenderAnimator"; |
| import {HighlightManager, type CharIntervalUnderlineSeg} from "./HighlightManager"; |
| import {SvgOverlayManager} from "./SvgOverlayManager"; |
| import {TokenPositionCalculator} from "./TokenPositionCalculator"; |
| import {ResizeHandler} from "./ResizeHandler"; |
| import {TokenFragmentRect, HighlightStyle} from "./types"; |
| import {ScrollbarMinimap} from "./ScrollbarMinimap"; |
| import {isNarrowScreen} from "../utils/responsive"; |
| import {getTokenRenderStyle} from "../utils/tokenRenderStyle"; |
| import {getInfoDensityRenderDisabled} from "../utils/infoDensityRenderManager"; |
| import type { FrontendToken } from "../api/GLTR_API"; |
|
|
| |
| |
| |
| function getMinimapWidthFromCSS(): number { |
| const value = getComputedStyle(document.documentElement) |
| .getPropertyValue('--minimap-width') |
| .trim(); |
|
|
| if (!value) { |
| console.warn('CSS 变量 --minimap-width 未定义,使用默认值 12px'); |
| return 12; |
| } |
|
|
| |
| const match = value.match(/^(\d+(?:\.\d+)?)px$/); |
| if (match) { |
| return parseFloat(match[1]); |
| } |
|
|
| console.warn(`CSS 变量 --minimap-width 格式无效: "${value}",使用默认值 12px`); |
| return 12; |
| } |
|
|
| export enum GLTR_Mode { |
| fract_p |
| } |
|
|
| |
| export type TokenDataForRender = FrontendToken & { rawScoreNormed?: number }; |
|
|
| |
| export type SemanticRenderFields = { |
| pwScore?: number; |
| |
| signalProb?: number; |
| rawScoreNormed?: number; |
| |
| rawScore?: number; |
| chunkIndex?: number; |
| chunkMatchDegree?: number; |
| }; |
|
|
| export type GLTR_RenderItem = { |
| tokenData: TokenDataForRender; |
| |
| semantic?: SemanticRenderFields; |
| }; |
| export type GLTR_HoverEvent = { hovered: boolean, d: GLTR_RenderItem, event?: MouseEvent } |
|
|
| |
| export type GLTR_TokenClickEvent = { |
| tokenIndex: number; |
| }; |
|
|
| |
| function extractSemanticFields(token: TokenDataForRender): SemanticRenderFields | undefined { |
| const rawScoreNormed = "rawScoreNormed" in token && typeof token.rawScoreNormed === "number" ? token.rawScoreNormed : undefined; |
| if (rawScoreNormed === undefined) return undefined; |
| return { rawScoreNormed }; |
| } |
|
|
| export class GLTR_Text_Box extends VComponent<FrontendAnalyzeResult> { |
| protected _current = { |
| maxValue: -1, |
| highlightedIndices: new Set<number>(), |
| highlightStyle: 'border' as 'border' | 'underline', |
| |
| chunkCharRange: null as { x0: number; x1: number } | null, |
| |
| diffMode: false, |
| deltaByteSurprisals: [] as number[], |
| charToByteIndexMap: [] as number[], |
| }; |
| protected css_name = "LMF"; |
| protected options = { |
| gltrMode: GLTR_Mode.fract_p, |
| diffScale: d3.scalePow<string>().exponent(.3).range(["#b4e876", "#fff"]), |
| fracScale: d3.scaleLinear<string>().domain([0, 15]).range(["#fff", "#ff8080"]), |
| |
| enableRenderAnimation: false, |
| |
| enableMinimap: false, |
| minimapWidth: getMinimapWidthFromCSS(), |
| |
| semanticAnalysisMode: false, |
| |
| surprisalColorMax: undefined as number | undefined, |
| |
| overlayTokenRenderStyle: undefined as 'density' | 'classic' | undefined, |
| |
| overlayIgnoreGlobalInfoDensityDisable: false, |
| |
| |
| |
| |
| overlayForceDisableInfoDensityRender: false, |
| |
| |
| |
| |
| onFullTextLayerRenderComplete: undefined as (() => void) | undefined, |
| }; |
|
|
| |
| private getOverlayDisableInfoDensityRender(): boolean { |
| if (this.options.overlayForceDisableInfoDensityRender) { |
| return true; |
| } |
| if (this.options.overlayIgnoreGlobalInfoDensityDisable) { |
| return false; |
| } |
| return getInfoDensityRenderDisabled(); |
| } |
|
|
| |
| |
| |
| private getThemeMode(): 'light' | 'dark' { |
| const isDark = document.documentElement.getAttribute('data-theme') === 'dark'; |
| return isDark ? 'dark' : 'light'; |
| } |
| |
| |
| private renderAnimator?: RenderAnimator; |
| |
| |
| private animationDelay: number = 100; |
| |
| |
| private currentRenderData?: FrontendAnalyzeResult; |
| |
| |
| private currentSvgOverlay?: SVGSVGElement; |
|
|
| |
| private _renderVersion = 0; |
| |
| |
| private resizeHandler?: ResizeHandler; |
|
|
| |
| private positionCalculator?: TokenPositionCalculator; |
|
|
| |
| private cachedPositions?: TokenFragmentRect[]; |
| private cachedPositionsText?: string; |
| private cachedPositionsTokenCount: number = 0; |
| |
| private cachedContainerWidth: number = 0; |
|
|
| |
| private svgOverlayManager?: SvgOverlayManager; |
| |
| |
| private highlightManager?: HighlightManager; |
| |
| |
| private underlineCache: Map<string, SVGLineElement> = new Map(); |
| |
| |
| private minimapManager?: ScrollbarMinimap; |
|
|
| private _refreshBaseRectColorsOrFullRender = (): void => { |
| if (this.svgOverlayManager && this.currentRenderData) { |
| const rectCount = this.svgOverlayManager.getRectCache().size; |
| const tokenCount = this.currentRenderData.bpe_strings.length; |
| |
| if (rectCount < tokenCount) { |
| this.reRenderCurrent(true); |
| return; |
| } |
| this.svgOverlayManager.updateBaseRectColors(this.currentRenderData, { |
| disableInfoDensityRender: this.getOverlayDisableInfoDensityRender(), |
| tokenRenderStyle: this.options.overlayTokenRenderStyle ?? getTokenRenderStyle(), |
| }); |
| } else { |
| this.reRenderCurrent(true); |
| } |
| }; |
| private _onTokenRenderStyleChange = (): void => this._refreshBaseRectColorsOrFullRender(); |
| private _onInfoDensityRenderChange = (): void => this._refreshBaseRectColorsOrFullRender(); |
|
|
| static events = { |
| tokenHovered: 'lmf-view-token-hovered', |
| |
| tokenClicked: 'lmf-view-token-clicked', |
| }; |
|
|
| |
| |
| |
| |
| getCurrentAnalyzeResult(): FrontendAnalyzeResult | null { |
| const rd = (this.currentRenderData ?? this.renderData) as FrontendAnalyzeResult | undefined; |
| if (!rd?.bpe_strings?.length) return null; |
| return rd; |
| } |
|
|
| |
| private notifyFullTextLayerRenderCompleteIfCurrent(myVersion: number): void { |
| if (myVersion !== this._renderVersion) return; |
| this.options.onFullTextLayerRenderComplete?.(); |
| } |
|
|
| constructor(parent: D3Sel, eventHandler?: SimpleEventHandler, options = {}) { |
| super(parent, eventHandler); |
| this.superInitHTML(options); |
| this._init(); |
| |
| |
| |
| |
| this.animationDelay = 100; |
| this.renderAnimator = new RenderAnimator({ |
| enabled: this.options.enableRenderAnimation, |
| delayBetweenBatches: this.animationDelay, |
| }); |
| |
| |
| this.setupThemeListener(); |
| window.addEventListener('token-render-style-change', this._onTokenRenderStyleChange); |
| window.addEventListener('info-density-render-change', this._onInfoDensityRenderChange); |
|
|
| |
| this.updateColorScales(); |
|
|
| |
| this.syncMinimapEnabledClass(); |
| } |
|
|
| protected _init() { |
| |
| this.createLoadingOverlay(); |
| } |
|
|
| |
| |
| |
| private createLoadingOverlay(): void { |
| const baseNode = this.base.node(); |
| if (!baseNode) return; |
|
|
| |
| if (baseNode.querySelector('.text-loading-overlay')) { |
| return; |
| } |
|
|
| const overlay = document.createElement('div'); |
| overlay.className = 'text-loading-overlay'; |
| overlay.innerHTML = ` |
| <div class="loading-content"> |
| <div class="loading-spinner"></div> |
| </div> |
| `; |
| baseNode.appendChild(overlay); |
| } |
|
|
|
|
| protected _render(rd: FrontendAnalyzeResult = this.renderData): void { |
| if (!rd) return; |
|
|
| this._renderVersion++; |
| |
| this.currentRenderData = rd; |
|
|
| |
|
|
| |
| if (this._current.diffMode && this._current.deltaByteSurprisals.length > 0) { |
| const originalText = rd.originalText; |
| this._current.charToByteIndexMap = buildCharToByteIndexMap(originalText); |
| } |
|
|
| |
| this.hideLoading(); |
|
|
| |
| |
| this._renderWithSvgOverlay(rd).catch(err => { |
| console.error('SVG渲染出错:', err); |
| }); |
| } |
|
|
| |
| |
| |
| |
| protected async _renderWithSvgOverlay(rd: FrontendAnalyzeResult): Promise<void> { |
| const myVersion = this._renderVersion; |
|
|
| const rdExt = rd as FrontendAnalyzeResult & { rawScoresNormed?: (number | undefined)[]; colorScores?: number[]; chunkInfos?: Array<{ startOffset: number }> }; |
| const rawScoresNormed = rdExt.rawScoresNormed; |
| const colorScores = (rdExt.colorScores?.length ? rdExt.colorScores : undefined) ?? rawScoresNormed; |
| const isSemantic = this.options.semanticAnalysisMode && colorScores?.length; |
|
|
| |
| |
| |
| const baseNodeForCheck = this.base.node(); |
| const svgInDOM = !!(this.currentSvgOverlay?.parentNode); |
| const isChunkedSemantic = Boolean(rdExt.chunkInfos?.length); |
| const nTok = rd.bpe_strings.length; |
| const cachedTok = this.cachedPositionsTokenCount; |
| const sameTokenCount = nTok === cachedTok; |
| const chunkAppendOnly = |
| isChunkedSemantic && nTok > cachedTok; |
| const canReuseSvgForIncremental = sameTokenCount || chunkAppendOnly; |
| const canIncremental = |
| isSemantic && |
| this.currentSvgOverlay && |
| svgInDOM && |
| this.cachedPositions && |
| this.svgOverlayManager && |
| canReuseSvgForIncremental && |
| rd.originalText === this.cachedPositionsText && |
| baseNodeForCheck != null && |
| baseNodeForCheck.clientWidth === this.cachedContainerWidth; |
|
|
| if (canIncremental) { |
| |
| this.updateTruncatedBoundary(rd); |
| |
| this.positionCalculator?.resetIndex(); |
|
|
| |
| |
| if (rd.bpe_strings.length > this.cachedPositionsTokenCount) { |
| const prevTokenCount = this.cachedPositionsTokenCount; |
| const newPositions = this.positionCalculator!.calculateTokenPositions(rd, prevTokenCount); |
| this.svgOverlayManager!.appendTokenRects(newPositions, this.currentSvgOverlay!, rd); |
| this.cachedPositions = [...(this.cachedPositions ?? []), ...newPositions]; |
| this.cachedPositionsTokenCount = rd.bpe_strings.length; |
| } |
|
|
| |
| const latestChunk = rdExt.chunkInfos?.[rdExt.chunkInfos.length - 1]; |
| const fromTokenIndex = latestChunk |
| ? Math.max(0, rd.bpe_strings.findIndex(t => t.offset[0] >= latestChunk.startOffset)) |
| : 0; |
| this.svgOverlayManager!.updateSemanticColors(colorScores!, fromTokenIndex); |
|
|
| |
| if (this.cachedPositions && this.cachedPositions.length > 0) { |
| await this.renderMinimap(this.cachedPositions, rd); |
| } |
| return; |
| } |
|
|
| |
| |
| this.clearVisualization(); |
|
|
| |
| this.setContainerText(rd); |
|
|
| |
| await new Promise(resolve => requestAnimationFrame(resolve)); |
| await new Promise(resolve => setTimeout(resolve, 10)); |
|
|
| if (myVersion !== this._renderVersion) return; |
|
|
| const baseNode = this.base.node(); |
| if (!baseNode) return; |
| |
| if (!this.positionCalculator) { |
| this.positionCalculator = new TokenPositionCalculator(baseNode); |
| } |
|
|
| const rdForPositions: FrontendAnalyzeResult = rd; |
| let positions = this.positionCalculator.calculateTokenPositions(rdForPositions); |
| if (isSemantic && rawScoresNormed?.length) { |
| const chunkInfos = (rd as FrontendAnalyzeResult & { chunkInfos?: unknown[] }).chunkInfos; |
| |
| if (!chunkInfos?.length) { |
| positions = positions.filter((p) => rawScoresNormed[p.tokenIndex] !== undefined); |
| } |
| } |
|
|
| if (positions.length === 0) { |
| |
| if (rd.bpe_strings.length > 0) { |
| console.warn('⚠️ 没有有效的token位置'); |
| } |
| this.notifyFullTextLayerRenderCompleteIfCurrent(myVersion); |
| return; |
| } |
|
|
| const overlayOptions = { |
| getTokenRealTopk: (r: FrontendAnalyzeResult, tokenIndex: number) => this.getTokenRealTopk(r, tokenIndex), |
| addTokenEventListeners: (element: SVGGElement, tokenIndex: number, r: FrontendAnalyzeResult) => this.addTokenEventListeners(element, tokenIndex, r), |
| tokenRenderStyle: this.options.overlayTokenRenderStyle ?? getTokenRenderStyle(), |
| disableInfoDensityRender: this.getOverlayDisableInfoDensityRender(), |
| diff: this._current.diffMode && this._current.deltaByteSurprisals.length > 0 |
| ? { |
| enabled: true, |
| deltaByteSurprisals: this._current.deltaByteSurprisals, |
| charToByteIndexMap: this._current.charToByteIndexMap, |
| } |
| : undefined, |
| semantic: this.options.semanticAnalysisMode ? { analysisMode: true, rawScoresNormed: colorScores } : undefined, |
| surprisalColorMax: this.options.surprisalColorMax, |
| }; |
| this.svgOverlayManager = new SvgOverlayManager(baseNode, overlayOptions); |
|
|
| const svg = this.svgOverlayManager.createSvgOverlay(positions, rdForPositions); |
|
|
| |
| this.highlightManager = new HighlightManager( |
| svg, |
| this.svgOverlayManager.getRectCache(), |
| this.underlineCache |
| ); |
|
|
| |
| if (myVersion !== this._renderVersion) return; |
|
|
| this.currentSvgOverlay = svg; |
| |
| baseNode.appendChild(svg); |
|
|
| |
| this.cachedPositions = positions; |
| this.cachedPositionsText = rd.originalText; |
| this.cachedPositionsTokenCount = rd.bpe_strings.length; |
| this.cachedContainerWidth = baseNode.clientWidth; |
|
|
| |
| this.setupResizeHandler(); |
|
|
| |
| if (this.renderAnimator && this.options.enableRenderAnimation) { |
| await this.animateSvgRects(svg, positions); |
| } |
|
|
| |
| const delay = this.renderAnimator && this.options.enableRenderAnimation ? 200 : 0; |
| if (this._current.chunkCharRange) { |
| const { x0, x1 } = this._current.chunkCharRange; |
| setTimeout(() => this.setChunkCharRangeHighlight(x0, x1), delay); |
| } else if (this._current.highlightedIndices.size > 0) { |
| setTimeout(() => { |
| this.setHighlightedIndices(this._current.highlightedIndices, this._current.highlightStyle); |
| }, delay); |
| } |
| |
| |
| await this.renderMinimap(positions, rd); |
| this.notifyFullTextLayerRenderCompleteIfCurrent(myVersion); |
| } |
|
|
| |
| |
| |
| |
| |
| private async animateSvgRects(svg: SVGSVGElement, positions: TokenFragmentRect[]): Promise<void> { |
| if (!this.renderAnimator) return; |
|
|
| const totalTokens = positions.length; |
| const initialBatchSize = 32; |
| let currentIndex = 0; |
| let currentBatchSize = initialBatchSize; |
|
|
| |
| const rectCache = this.svgOverlayManager?.getRectCache(); |
| const overlayCache = this.svgOverlayManager?.getSemanticOverlayCache(); |
| if (rectCache) { |
| rectCache.forEach(({ rect }) => rect.setAttribute('fill-opacity', '0')); |
| } |
| overlayCache?.forEach((rect) => rect.setAttribute('fill-opacity', '0')); |
|
|
| |
| await new Promise(resolve => setTimeout(resolve, this.animationDelay)); |
|
|
| while (currentIndex < totalTokens) { |
| const actualBatchSize = Math.min(currentBatchSize, totalTokens - currentIndex); |
| for (let i = currentIndex; i < currentIndex + actualBatchSize; i++) { |
| const rectKey = positions[i].rectKey; |
| rectCache?.get(rectKey)?.rect?.setAttribute('fill-opacity', '1'); |
| overlayCache?.get(rectKey)?.setAttribute('fill-opacity', '1'); |
| } |
|
|
| currentIndex += actualBatchSize; |
|
|
| |
| if (currentIndex < totalTokens) { |
| await new Promise(resolve => setTimeout(resolve, this.animationDelay)); |
| currentBatchSize = Math.floor(currentBatchSize * 1.5); |
| } |
| } |
| } |
|
|
|
|
| |
| |
| |
| |
| private computeTruncatedLength( |
| tokens: Array<{ offset: [number, number] }>, |
| rawScores?: (number | undefined)[], |
| chunkInfos?: Array<{ startOffset: number; endOffset: number }> |
| ): number { |
| |
| if (chunkInfos?.length) { |
| return chunkInfos[chunkInfos.length - 1]!.endOffset; |
| } |
| |
| if (rawScores?.length && tokens.length > 0) { |
| let lastIdx = -1; |
| for (let i = rawScores.length - 1; i >= 0; i--) { |
| if (rawScores[i] !== undefined) { |
| lastIdx = i; |
| break; |
| } |
| } |
| if (lastIdx >= 0) return tokens[lastIdx]!.offset[1]; |
| } |
| |
| return tokens.length > 0 ? tokens[tokens.length - 1]!.offset[1] : 0; |
| } |
|
|
| |
| |
| |
| |
| private updateTruncatedBoundary(rd: FrontendAnalyzeResult): void { |
| const baseNode = this.base.node(); |
| if (!baseNode) return; |
| const textLayer = baseNode.querySelector('.text-layer') as HTMLElement | null; |
| if (!textLayer) return; |
|
|
| const rdExt = rd as FrontendAnalyzeResult & { |
| rawScoresNormed?: (number | undefined)[]; |
| chunkInfos?: Array<{ startOffset: number; endOffset: number }>; |
| }; |
| const truncatedLength = this.computeTruncatedLength(rd.bpe_strings, rdExt.rawScoresNormed, rdExt.chunkInfos); |
| const fullText = rd.originalText; |
| const isTruncated = truncatedLength < fullText.length; |
|
|
| const textNode = textLayer.firstChild; |
| if (textNode && textNode.nodeType === Node.TEXT_NODE) { |
| const expected = isTruncated ? fullText.slice(0, truncatedLength) : fullText; |
| if (textNode.textContent !== expected) textNode.textContent = expected; |
| } |
|
|
| const span = textLayer.querySelector('.truncated-text') as HTMLElement | null; |
| const remaining = isTruncated ? fullText.slice(truncatedLength) : ''; |
| if (remaining) { |
| if (span) { |
| if (span.textContent !== remaining) span.textContent = remaining; |
| } else { |
| const newSpan = document.createElement('span'); |
| newSpan.className = 'truncated-text'; |
| newSpan.textContent = remaining; |
| textLayer.appendChild(newSpan); |
| } |
| } else if (span) { |
| span.remove(); |
| } |
| } |
|
|
| |
| |
| |
| |
| |
| private setContainerText(rd: FrontendAnalyzeResult): void { |
| const baseNode = this.base.node(); |
| if (!baseNode) return; |
|
|
| |
| while (baseNode.firstChild) { |
| baseNode.removeChild(baseNode.firstChild); |
| } |
|
|
| const fullText = rd.originalText; |
| if (!fullText) { |
| if (baseNode) { |
| if (!this.positionCalculator) { |
| this.positionCalculator = new TokenPositionCalculator(baseNode); |
| } else { |
| this.positionCalculator.resetIndex(); |
| } |
| } |
| return; |
| } |
|
|
| const textContainer = document.createElement('div'); |
| textContainer.className = 'text-layer'; |
| textContainer.style.position = 'relative'; |
| textContainer.style.zIndex = '2'; |
|
|
| const tokens = rd.bpe_strings; |
| const rawScores = (rd as FrontendAnalyzeResult & { rawScoresNormed?: (number | undefined)[] }).rawScoresNormed; |
| const chunkInfos = (rd as FrontendAnalyzeResult & { chunkInfos?: Array<{ startOffset: number; endOffset: number }> }).chunkInfos; |
| const truncatedLength = this.computeTruncatedLength(tokens, rawScores, chunkInfos); |
| const isTruncated = truncatedLength < fullText.length; |
|
|
| if (isTruncated) { |
| textContainer.appendChild(document.createTextNode(fullText.slice(0, truncatedLength))); |
| const span = document.createElement('span'); |
| span.className = 'truncated-text'; |
| span.textContent = fullText.slice(truncatedLength); |
| textContainer.appendChild(span); |
| } else { |
| textContainer.appendChild(document.createTextNode(fullText)); |
| } |
|
|
| baseNode.appendChild(textContainer); |
|
|
| if (baseNode) { |
| if (!this.positionCalculator) { |
| this.positionCalculator = new TokenPositionCalculator(baseNode); |
| } else { |
| this.positionCalculator.resetIndex(); |
| } |
| } |
| } |
|
|
| |
| |
| |
| |
| |
| public setTextOnly(text: string): void { |
| const baseNode = this.base.node(); |
| if (!baseNode) return; |
|
|
| |
| if (this.options.enableMinimap && this.minimapManager) { |
| this.minimapManager.clear(); |
| } |
|
|
| |
| const existingOverlay = baseNode.querySelector('.text-loading-overlay'); |
|
|
| |
| while (baseNode.firstChild) { |
| baseNode.removeChild(baseNode.firstChild); |
| } |
|
|
| |
| this.currentSvgOverlay = undefined; |
| this.cachedPositions = undefined; |
| this.cachedPositionsText = undefined; |
| this.cachedPositionsTokenCount = 0; |
| this.cachedContainerWidth = 0; |
| this.svgOverlayManager?.clearRectCache(); |
|
|
| |
| if (text) { |
| const textContainer = document.createElement('div'); |
| textContainer.className = 'text-layer'; |
| textContainer.style.position = 'relative'; |
| textContainer.style.zIndex = '2'; |
| const textNode = document.createTextNode(text); |
| textContainer.appendChild(textNode); |
| baseNode.appendChild(textContainer); |
| } |
|
|
| |
| if (existingOverlay) { |
| baseNode.appendChild(existingOverlay); |
| } else { |
| this.createLoadingOverlay(); |
| } |
|
|
| |
| this.showLoading(); |
| } |
|
|
| |
| |
| |
| public showLoading(): void { |
| const baseNode = this.base.node(); |
| if (!baseNode) return; |
|
|
| |
| baseNode.classList.add('loading'); |
|
|
| |
| const overlay = baseNode.querySelector('.text-loading-overlay') as HTMLElement; |
| if (overlay) { |
| overlay.classList.add('visible'); |
| } |
| } |
|
|
| |
| |
| |
| public hideLoading(): void { |
| const baseNode = this.base.node(); |
| if (!baseNode) return; |
|
|
| |
| baseNode.classList.remove('loading'); |
|
|
| |
| const overlay = baseNode.querySelector('.text-loading-overlay') as HTMLElement; |
| if (overlay) { |
| overlay.classList.remove('visible'); |
| } |
| } |
|
|
| |
| |
| |
| private clearVisualization(): void { |
| const baseNode = this.base.node(); |
| if (baseNode) { |
| |
| const svgOverlay = baseNode.querySelector('.svg-overlay'); |
| if (svgOverlay) { |
| svgOverlay.remove(); |
| } |
| |
| |
| this.currentSvgOverlay = undefined; |
|
|
| |
| this.cachedPositions = undefined; |
| this.cachedPositionsText = undefined; |
| this.cachedPositionsTokenCount = 0; |
| this.cachedContainerWidth = 0; |
|
|
| |
| this.svgOverlayManager?.clearRectCache(); |
| |
| |
| this.underlineCache.clear(); |
|
|
| |
| if (this.options.enableMinimap && this.minimapManager) { |
| this.minimapManager.clear(); |
| } |
| |
| |
| if (!baseNode.querySelector('.text-loading-overlay')) { |
| this.createLoadingOverlay(); |
| } |
| } |
| } |
| |
| |
| |
| |
| private setupResizeHandler(): void { |
| |
| if (this.resizeHandler) { |
| return; |
| } |
| |
| const baseNode = this.base.node(); |
| if (!baseNode) return; |
| |
| |
| this.resizeHandler = new ResizeHandler(baseNode, { |
| onPositionUpdate: () => this.updateSvgPositions(), |
| getCurrentSvg: () => this.currentSvgOverlay, |
| onTransitionStart: () => { |
| |
| if (this.options.enableMinimap && this.minimapManager) { |
| this.minimapManager.hide(); |
| } |
| }, |
| }); |
| |
| |
| this.resizeHandler.setup(); |
| } |
| |
| |
| |
| |
| |
| private updateSvgPositions(): void { |
| if (!this.currentSvgOverlay || !this.currentRenderData) { |
| return; |
| } |
|
|
| const baseNode = this.base.node(); |
| if (!baseNode) return; |
| |
| |
| if (!this.positionCalculator) { |
| this.positionCalculator = new TokenPositionCalculator(baseNode); |
| } |
| const positions = this.positionCalculator.calculateTokenPositions(this.currentRenderData); |
| |
| if (positions.length === 0) { |
| return; |
| } |
|
|
| |
| if (!this.svgOverlayManager || this.svgOverlayManager.hasMissingRects(positions)) { |
| this._renderWithSvgOverlay(this.currentRenderData).catch(err => { |
| console.error('SVG渲染出错:', err); |
| }); |
| return; |
| } |
| |
| |
| this.svgOverlayManager.updateSvgPositions(this.currentSvgOverlay, positions); |
|
|
| |
| this.cachedPositions = positions; |
| this.cachedPositionsText = this.currentRenderData.originalText; |
| this.cachedPositionsTokenCount = this.currentRenderData.bpe_strings.length; |
| this.cachedContainerWidth = baseNode.clientWidth; |
|
|
| |
| this.highlightManager?.updateUnderlinePositions(); |
|
|
| this.refreshChunkCharRangeUnderlines(); |
|
|
| |
| if (this.options.enableMinimap && this.minimapManager) { |
| this.renderMinimap(positions, this.currentRenderData).catch(err => { |
| console.error('Minimap渲染出错:', err); |
| }); |
| } |
| } |
|
|
|
|
| |
|
|
|
|
| |
|
|
| |
| |
| |
| |
| |
| |
| public reRenderCurrent(forceFullRender = false): void { |
| if (!this.currentRenderData) return; |
| if (forceFullRender) { |
| this.cachedPositions = undefined; |
| this.cachedPositionsText = undefined; |
| this.cachedPositionsTokenCount = 0; |
| this.cachedContainerWidth = 0; |
| } |
| const wasAnimation = this.options.enableRenderAnimation; |
| this.options.enableRenderAnimation = false; |
| this._render(this.currentRenderData); |
| setTimeout(() => { this.options.enableRenderAnimation = wasAnimation; }, 0); |
| } |
|
|
| |
| |
| |
| |
| private updateColorScales(): void { |
| const theme = this.getThemeMode(); |
| |
| |
| |
| |
| const fracStartColor = theme === 'dark' ? "#191919" : "#fff"; |
| this.options.fracScale.range([fracStartColor, "#ff8080"]); |
| |
| |
| |
| |
| const diffEndColor = theme === 'dark' ? "#191919" : "#fff"; |
| this.options.diffScale.range(["#b4e876", diffEndColor]); |
| } |
| |
| |
| |
| |
| |
| private setupThemeListener(): void { |
| const observer = new MutationObserver((mutations) => { |
| mutations.forEach((mutation) => { |
| if (mutation.type === 'attributes' && mutation.attributeName === 'data-theme') { |
| this.updateColorScales(); |
| } |
| }); |
| }); |
| observer.observe(document.documentElement, { |
| attributes: true, |
| attributeFilter: ['data-theme'] |
| }); |
| } |
|
|
| |
| |
| |
| private getTokenRealTopk(rd: FrontendAnalyzeResult, tokenIndex: number): [number, number] | undefined { |
| const token = rd.bpe_strings[tokenIndex]; |
| return token.real_topk |
| ? token.real_topk as [number, number] |
| : undefined; |
| } |
|
|
| |
| |
| |
| |
| private addTokenEventListeners(element: SVGGElement, tokenIndex: number, rd: FrontendAnalyzeResult): void { |
| const tokenData = rd.bpe_strings[tokenIndex] as TokenDataForRender; |
| |
| const computeSemantic = (): SemanticRenderFields | undefined => { |
| |
| const latestRd = this.currentRenderData ?? rd; |
| const latestExt = latestRd as FrontendAnalyzeResult & { |
| rawScoresNormed?: number[]; |
| attentionRawScores?: number[]; |
| pPwValues?: number[]; |
| pwScores?: number[]; |
| }; |
|
|
| const rawScoresNormed = latestExt.rawScoresNormed; |
| const hasRawScoresNormedNow = rawScoresNormed?.length && tokenIndex < rawScoresNormed.length; |
|
|
| let semantic = extractSemanticFields(tokenData); |
| if (hasRawScoresNormedNow && rawScoresNormed) { |
| |
| const attnScore = rawScoresNormed[tokenIndex]; |
| const rawScore = latestExt.attentionRawScores?.[tokenIndex]; |
| const signalProb = latestExt.pPwValues?.[tokenIndex]; |
| const pwScore = latestExt.pwScores?.[tokenIndex]; |
|
|
| const tokenOffset = |
| latestRd.bpeBpeMergedTokens?.[tokenIndex]?.offset ?? latestRd.bpe_strings[tokenIndex]?.offset; |
| const rdChunkInfos = (latestRd as FrontendAnalyzeResult & { |
| chunkInfos?: Array<{ startOffset: number; endOffset: number; chunkIndex?: number; chunkMatchDegree?: number }>; |
| }).chunkInfos; |
| const chunkInfo = tokenOffset && rdChunkInfos?.find( |
| c => tokenOffset[0] >= c.startOffset && tokenOffset[0] < c.endOffset |
| ); |
|
|
| semantic = { |
| ...semantic, |
| rawScoreNormed: attnScore, |
| rawScore, |
| signalProb, |
| pwScore, |
| chunkIndex: chunkInfo?.chunkIndex, |
| chunkMatchDegree: chunkInfo?.chunkMatchDegree, |
| } as SemanticRenderFields; |
| } |
| return semantic; |
| }; |
|
|
| const handleMouseEnter = (event: MouseEvent) => { |
| |
| if ((event.buttons & 1) !== 0) return; |
| this.eventHandler.trigger(GLTR_Text_Box.events.tokenHovered, <GLTR_HoverEvent>{ |
| hovered: true, |
| d: { tokenData, semantic: computeSemantic() }, |
| event: event |
| }); |
| |
| }; |
|
|
| const handleMouseLeave = (event: MouseEvent) => { |
| this.eventHandler.trigger(GLTR_Text_Box.events.tokenHovered, <GLTR_HoverEvent>{ |
| hovered: false, |
| |
| d: { tokenData, semantic: undefined }, |
| event: event |
| }); |
| }; |
|
|
| |
| const handleMouseDown = (event: MouseEvent) => { |
| if (event.button !== 0) return; |
| this.eventHandler.trigger(GLTR_Text_Box.events.tokenHovered, <GLTR_HoverEvent>{ |
| hovered: false, |
| d: { tokenData, semantic: undefined }, |
| event: event |
| }); |
| }; |
|
|
| element.addEventListener('mouseenter', handleMouseEnter); |
| element.addEventListener('mouseleave', handleMouseLeave); |
| element.addEventListener('mousedown', handleMouseDown); |
|
|
| element.addEventListener('click', (event: MouseEvent) => { |
| if (event.button !== 0) return; |
| event.stopPropagation(); |
| this.eventHandler.trigger(GLTR_Text_Box.events.tokenClicked, <GLTR_TokenClickEvent>{ |
| tokenIndex, |
| }); |
| }); |
| } |
| |
| |
| |
| |
| private async renderMinimap(positions: TokenFragmentRect[], rd: FrontendAnalyzeResult): Promise<void> { |
| if (!this.options.enableMinimap) { |
| return; |
| } |
|
|
| this.ensureMinimapManager(); |
| if (!this.minimapManager) return; |
|
|
| |
| if (positions.length === 0) { |
| this.minimapManager.clear(); |
| return; |
| } |
|
|
| await this.minimapManager.render(positions, rd, { |
| semanticAnalysisMode: this.options.semanticAnalysisMode, |
| measureCharRangeY: (startOffset: number, endOffset: number) => |
| this.measureCharRangeY(startOffset, endOffset), |
| }); |
| } |
|
|
| private measureCharRangeY(startOffset: number, endOffset: number): { minY: number; maxY: number } | null { |
| const baseNode = this.base.node(); |
| if (!baseNode || endOffset <= startOffset) return null; |
|
|
| const calculator = this.positionCalculator ?? new TokenPositionCalculator(baseNode); |
| const start = calculator.findNodeAndOffset(Math.max(0, startOffset)); |
| const end = calculator.findNodeAndOffset(Math.max(0, endOffset)); |
| if (!start || !end) return null; |
|
|
| const range = document.createRange(); |
| range.setStart(start.node, start.offset); |
| range.setEnd(end.node, end.offset); |
|
|
| const containerRect = baseNode.getBoundingClientRect(); |
| const zoom = calculator.getZoom(); |
| let minY = Number.POSITIVE_INFINITY; |
| let maxY = Number.NEGATIVE_INFINITY; |
| for (const rect of range.getClientRects()) { |
| if (rect.width === 0 && rect.height === 0) continue; |
| const top = (rect.top - containerRect.top) / zoom; |
| const bottom = (rect.bottom - containerRect.top) / zoom; |
| minY = Math.min(minY, top); |
| maxY = Math.max(maxY, bottom); |
| } |
| return Number.isFinite(minY) && Number.isFinite(maxY) ? { minY, maxY } : null; |
| } |
|
|
| |
| |
| |
| |
| |
| private calculateTraditionalScrollbarWidth(): number { |
| const baseNode = this.base.node(); |
| if (!baseNode) { |
| return 0; |
| } |
|
|
| const rightPanel = document.querySelector('.right_panel') as HTMLElement; |
| if (!rightPanel) { |
| return 0; |
| } |
|
|
| |
| const rightPanelWidth = rightPanel.offsetWidth; |
| |
| const lmfWidth = baseNode.offsetWidth; |
| |
| const scrollbarWidth = rightPanelWidth - lmfWidth; |
|
|
| |
| return scrollbarWidth > 0 ? scrollbarWidth : 0; |
| } |
|
|
| |
| |
| |
| private ensureMinimapManager(): void { |
| const baseNode = this.base.node(); |
| if (!baseNode) return; |
|
|
| |
| let minimapWidth: number = this.options.minimapWidth; |
|
|
| if (!isNarrowScreen()) { |
| |
| const scrollbarWidth = this.calculateTraditionalScrollbarWidth(); |
|
|
| if (scrollbarWidth > 0) { |
| |
| minimapWidth = scrollbarWidth; |
| } |
|
|
| |
| if (minimapWidth > 1) { |
| minimapWidth -= 1; |
| } |
| } |
|
|
| const config = { |
| width: minimapWidth |
| }; |
|
|
| if (!this.minimapManager) { |
| this.minimapManager = new ScrollbarMinimap(baseNode, config); |
| } else { |
| this.minimapManager.updateOptions(config); |
| } |
| } |
|
|
| |
| |
| |
| private syncMinimapEnabledClass(): void { |
| const baseNode = this.base.node(); |
| if (baseNode) { |
| baseNode.classList.toggle('minimap-enabled', this.options.enableMinimap); |
| } |
| } |
|
|
| |
| |
| |
| destroy(): void { |
| |
| if (this.resizeHandler) { |
| this.resizeHandler.destroy(); |
| this.resizeHandler = undefined; |
| } |
| |
| |
| if (this.minimapManager) { |
| this.minimapManager.destroy(); |
| this.minimapManager = undefined; |
| } |
| |
| |
| this.currentSvgOverlay = undefined; |
| window.removeEventListener('token-render-style-change', this._onTokenRenderStyleChange); |
| window.removeEventListener('info-density-render-change', this._onInfoDensityRenderChange); |
|
|
| |
| super.destroy(); |
| } |
|
|
| protected _wrangle(data: FrontendAnalyzeResult) { |
| const tokens = data.bpe_strings; |
| const allTop1 = tokens |
| .map(token => token.pred_topk.length > 0 ? token.pred_topk[0][1] : null) |
| .filter((value): value is number => typeof value === 'number' && Number.isFinite(value)); |
|
|
| if (allTop1.length === 0) { |
| |
| this._current.maxValue = 0; |
| this.options.diffScale.domain([0, 1]); |
| return data; |
| } |
|
|
| const maxTop1 = d3.max(allTop1); |
| this._current.maxValue = maxTop1 ?? 0; |
| this.options.diffScale.domain([0, this._current.maxValue || 1]); |
|
|
| return data; |
| } |
|
|
| |
| |
| |
| updateOptions(options: any, reRender = false) { |
| |
| if (options.hasOwnProperty('enableRenderAnimation') && this.renderAnimator) { |
| this.renderAnimator.setEnabled(options.enableRenderAnimation); |
| } |
|
|
| |
| const previousEnableMinimap = this.options.enableMinimap; |
|
|
| |
| super.updateOptions(options, reRender); |
|
|
| |
| if (options.hasOwnProperty('enableMinimap')) { |
| this.syncMinimapEnabledClass(); |
| |
| |
| if (this.currentRenderData && this.positionCalculator) { |
| if (this.options.enableMinimap && !previousEnableMinimap) { |
| |
| const positions = this.positionCalculator.calculateTokenPositions(this.currentRenderData); |
| if (positions.length > 0) { |
| this.renderMinimap(positions, this.currentRenderData).catch(err => { |
| console.error('Minimap渲染出错:', err); |
| }); |
| } |
| } else if (!this.options.enableMinimap && previousEnableMinimap && this.minimapManager) { |
| |
| this.minimapManager.destroy(); |
| this.minimapManager = undefined; |
| } |
| } |
| return; |
| } |
|
|
| |
| if (this.options.enableMinimap && this.minimapManager && this.currentRenderData && this.positionCalculator) { |
| const positions = this.positionCalculator.calculateTokenPositions(this.currentRenderData); |
| this.renderMinimap(positions, this.currentRenderData).catch(err => { |
| console.error('Minimap渲染出错:', err); |
| }); |
| } |
| } |
|
|
| |
| |
| |
| setChunkCharRangeHighlight(x0: number, x1: number): void { |
| const x0i = Math.max(0, Math.floor(x0)); |
| const x1i = Math.max(0, Math.floor(x1)); |
| if (x1i <= x0i) { |
| this._current.chunkCharRange = null; |
| this.highlightManager?.clearCharIntervalUnderlines(); |
| return; |
| } |
|
|
| this._current.chunkCharRange = { x0: x0i, x1: x1i }; |
| this._current.highlightedIndices.clear(); |
|
|
| if (!this.highlightManager) { |
| setTimeout(() => this.setChunkCharRangeHighlight(x0i, x1i), 50); |
| return; |
| } |
|
|
| const segments = this.computeCharIntervalUnderlineSegments(x0i, x1i); |
| this.highlightManager.setCharIntervalUnderlines(segments); |
| } |
|
|
| |
| private static clientRectToUnderlineSeg( |
| r: DOMRectReadOnly, |
| containerRect: DOMRectReadOnly, |
| zoom: number |
| ): CharIntervalUnderlineSeg { |
| return { |
| x1: (r.left - containerRect.left) / zoom, |
| x2: (r.right - containerRect.left) / zoom, |
| y: (r.bottom - containerRect.top) / zoom, |
| }; |
| } |
|
|
| private computeCharIntervalUnderlineSegments(x0: number, x1: number): CharIntervalUnderlineSeg[] { |
| const baseNode = this.base.node(); |
| if (!baseNode) { |
| throw new Error('[GLTR_Text_Box] chunk 下划线:缺少 base 节点'); |
| } |
| if (x1 <= x0) return []; |
|
|
| const calculator = this.positionCalculator ?? new TokenPositionCalculator(baseNode); |
| const a = calculator.findNodeAndOffset(x0); |
| const b = calculator.findNodeAndOffset(x1); |
| if (!a || !b) { |
| throw new Error( |
| `[GLTR_Text_Box] chunk 下划线:无法将 Unicode 半开区间 [${x0}, ${x1}) 映射到文本节点` |
| ); |
| } |
|
|
| const range = document.createRange(); |
| range.setStart(a.node, a.offset); |
| range.setEnd(b.node, b.offset); |
|
|
| const cr = baseNode.getBoundingClientRect(); |
| const z = calculator.getZoom(); |
| const toSeg = (box: DOMRectReadOnly) => GLTR_Text_Box.clientRectToUnderlineSeg(box, cr, z); |
|
|
| const segments: CharIntervalUnderlineSeg[] = []; |
| for (const r of range.getClientRects()) { |
| if (r.width !== 0 || r.height !== 0) { |
| segments.push(toSeg(r)); |
| } |
| } |
| if (segments.length === 0) { |
| throw new Error( |
| `[GLTR_Text_Box] chunk 下划线 [${x0}, ${x1}):` + |
| 'Range.getClientRects() 未产生任何有效矩形(不做包围盒回退)' |
| ); |
| } |
| return segments; |
| } |
|
|
| private refreshChunkCharRangeUnderlines(): void { |
| const c = this._current.chunkCharRange; |
| if (!c || !this.highlightManager) return; |
| this.highlightManager.updateCharIntervalUnderlines( |
| this.computeCharIntervalUnderlineSegments(c.x0, c.x1) |
| ); |
| } |
|
|
| |
| scrollToUnicodeCharOffset(charOffset: number): void { |
| requestAnimationFrame(() => { |
| const baseNode = this.base.node(); |
| if (!baseNode) return; |
|
|
| const calculator = this.positionCalculator ?? new TokenPositionCalculator(baseNode); |
| const safeOffset = Math.max(0, Math.floor(charOffset)); |
| const found = calculator.findNodeAndOffset(safeOffset); |
| if (!found) return; |
|
|
| const range = document.createRange(); |
| range.setStart(found.node, found.offset); |
| range.collapse(true); |
|
|
| let rect = range.getBoundingClientRect(); |
| if (rect.width === 0 && rect.height === 0) { |
| const rects = range.getClientRects(); |
| if (!rects.length) return; |
| rect = rects[0]; |
| } |
|
|
| const marginRatio = 0.2; |
| if (isNarrowScreen()) { |
| const y = window.scrollY + rect.top - window.innerHeight * marginRatio; |
| window.scrollTo({ top: Math.max(0, y), behavior: 'smooth' }); |
| return; |
| } |
|
|
| const panel = document.querySelector('.right_panel') as HTMLElement | null; |
| if (!panel) return; |
|
|
| const panelRect = panel.getBoundingClientRect(); |
| const topInPanel = rect.top - panelRect.top + panel.scrollTop; |
| const target = topInPanel - panel.clientHeight * marginRatio; |
| const maxScroll = Math.max(0, panel.scrollHeight - panel.clientHeight); |
| panel.scrollTo({ top: Math.max(0, Math.min(target, maxScroll)), behavior: 'smooth' }); |
| }); |
| } |
|
|
| |
| |
| |
| |
| |
| setHighlightedIndices(indices: Set<number>, highlightStyle: HighlightStyle = 'border') { |
| this._current.chunkCharRange = null; |
| this._current.highlightedIndices = indices; |
| this._current.highlightStyle = highlightStyle; |
| |
| if (!this.highlightManager) { |
| |
| setTimeout(() => { |
| this.setHighlightedIndices(indices, highlightStyle); |
| }, 50); |
| return; |
| } |
|
|
| this.highlightManager.setHighlightedIndices(indices, highlightStyle); |
| } |
|
|
| |
| |
| |
| clearHighlight() { |
| this._current.highlightedIndices.clear(); |
| this._current.chunkCharRange = null; |
|
|
| if (this.highlightManager) { |
| this.highlightManager.clearHighlight(); |
| } |
| } |
|
|
| |
| |
| |
| |
| |
| setDiffMode(enabled: boolean, deltaByteSurprisals: number[] = []) { |
| this._current.diffMode = enabled; |
| this._current.deltaByteSurprisals = deltaByteSurprisals; |
| |
| |
| if (this.currentRenderData) { |
| |
| |
| const originalText = this.currentRenderData.originalText; |
| this._current.charToByteIndexMap = buildCharToByteIndexMap(originalText); |
| |
| |
| const originalAnimationSetting = this.options.enableRenderAnimation; |
| this.options.enableRenderAnimation = false; |
| this._render(this.currentRenderData); |
| setTimeout(() => { |
| this.options.enableRenderAnimation = originalAnimationSetting; |
| }, 100); |
| } |
| } |
|
|
|
|
| } |