extension/splitTextToChunks.js

/**
 * SYNC 副本（demo 不抽共享包；改站内源请同步本文件）：
 * client/src/shared/cross/semanticUtils.ts → splitTextToChunks 及相关纯函数
 */
globalThis.IL_splitTextToChunks = (function () {
  const encoder = new TextEncoder();
  const YIELD_MS = 8;

  function yieldToMain(work) {
    const fn = globalThis.IL_yieldToMain;
    if (typeof fn !== 'function') {
      throw new Error('IL_yieldToMain missing — inject collectTextMap.js first');
    }
    return fn(work);
  }

  function getUtf8ByteLength(text, buf) {
    const { read, written } = encoder.encodeInto(text, buf);
    return read < text.length ? buf.length : written;
  }

  function nextParagraphEnd(text, start) {
    const nl = text.indexOf('\n\n', start);
    if (nl === -1) return text.length;
    let end = nl + 2;
    while (end < text.length && text[end] === '\n') end++;
    return end;
  }

  function nextLineEnd(text, start) {
    const nl = text.indexOf('\n', start);
    if (nl === -1) return text.length;
    let end = nl + 1;
    while (end < text.length && text[end] === '\n') end++;
    return end;
  }

  function charIndexForByteLimit(text, start, byteLimit) {
    const buf = new Uint8Array(4);
    let bytes = 0;
    let i = start;
    while (i < text.length) {
      const cp = text.codePointAt(i);
      const charLen = cp > 0xffff ? 2 : 1;
      const byteLen = encoder.encodeInto(text.slice(i, i + charLen), buf).written;
      if (bytes + byteLen > byteLimit) break;
      bytes += byteLen;
      i += charLen;
    }
    return i;
  }

  const SEPARATOR_GROUPS = [
    ['。', '！', '？', '…'],
    ['；', '，'],
    ['.', '!', '?'],
    [';', ','],
    [' ', '\t'],
  ];

  function findSplitPoint(text, start, maxEnd) {
    const window = text.slice(start, maxEnd);
    for (const group of SEPARATOR_GROUPS) {
      let bestEnd = -1;
      for (const sep of group) {
        const i = window.lastIndexOf(sep);
        if (i !== -1 && i + sep.length > bestEnd) bestEnd = i + sep.length;
      }
      if (bestEnd !== -1) return start + bestEnd;
    }
    return maxEnd;
  }

  function assertSplitArgs(text, bytesPerChunk) {
    if (bytesPerChunk <= 0) {
      throw new Error('bytesPerChunk must be > 0, got: ' + bytesPerChunk);
    }
    if (text.includes('\r')) {
      throw new Error('Text contains \\r (CR); only \\n (LF) is supported.');
    }
  }

  /** 从 pos 切出下一块的结束下标（不含）。 */
  function takeChunkEnd(text, pos, bytesPerChunk, encodeBuf) {
    let chunkEnd = pos;
    let chunkBytes = 0;
    outer: while (chunkEnd < text.length) {
      const paragEnd = nextParagraphEnd(text, chunkEnd);
      const paragBytes = getUtf8ByteLength(text.slice(chunkEnd, paragEnd), encodeBuf);
      if (chunkBytes > 0 && chunkBytes + paragBytes > bytesPerChunk) break;
      if (chunkBytes === 0 && paragBytes > bytesPerChunk) {
        while (chunkEnd < paragEnd) {
          const lineEnd = nextLineEnd(text, chunkEnd);
          const lineBytes = getUtf8ByteLength(text.slice(chunkEnd, lineEnd), encodeBuf);
          if (lineBytes > bytesPerChunk) {
            // 本 chunk 可能已累计 chunkBytes，超长行只能占剩余额度，否则整块超限
            const maxEnd = charIndexForByteLimit(text, chunkEnd, bytesPerChunk - chunkBytes);
            chunkEnd = findSplitPoint(text, chunkEnd, maxEnd);
            break outer;
          }
          if (chunkBytes > 0 && chunkBytes + lineBytes > bytesPerChunk) break outer;
          chunkBytes += lineBytes;
          chunkEnd = lineEnd;
        }
        continue outer;
      }
      chunkBytes += paragBytes;
      chunkEnd = paragEnd;
    }
    return chunkEnd;
  }

  /** @returns {{ text: string, startOffset: number }[]} */
  function splitTextToChunks(text, bytesPerChunk) {
    assertSplitArgs(text, bytesPerChunk);
    const chunks = [];
    let pos = 0;
    const encodeBuf = new Uint8Array(bytesPerChunk + 1);
    while (pos < text.length) {
      const chunkEnd = takeChunkEnd(text, pos, bytesPerChunk, encodeBuf);
      chunks.push({ text: text.slice(pos, chunkEnd), startOffset: pos });
      pos = chunkEnd;
    }
    return chunks;
  }

  /**
   * 与 splitTextToChunks 同结果；每 YIELD_MS 让出主线程。
   * @param {string} text
   * @param {number} bytesPerChunk
   * @param {() => boolean} [isStale]
   */
  async function splitTextToChunksAsync(text, bytesPerChunk, isStale) {
    assertSplitArgs(text, bytesPerChunk);
    const chunks = [];
    let pos = 0;
    const encodeBuf = new Uint8Array(bytesPerChunk + 1);
    while (pos < text.length) {
      await yieldToMain(() => {
        if (isStale?.()) {
          throw new DOMException('The operation was aborted.', 'AbortError');
        }
        const t0 = performance.now();
        while (pos < text.length && performance.now() - t0 < YIELD_MS) {
          if (isStale?.()) {
            throw new DOMException('The operation was aborted.', 'AbortError');
          }
          const chunkEnd = takeChunkEnd(text, pos, bytesPerChunk, encodeBuf);
          chunks.push({ text: text.slice(pos, chunkEnd), startOffset: pos });
          pos = chunkEnd;
        }
      });
    }
    return chunks;
  }

  globalThis.IL_splitTextToChunksAsync = splitTextToChunksAsync;
  return splitTextToChunks;
})();