File size: 2,228 Bytes
c126239 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 | /**
* Sentence segmentation via Intl.Segmenter (replaces nltk sent_tokenize used
* by the SemStamp reference implementation).
*/
import { browserLocale } from './locale';
export interface SentenceSpan {
text: string;
start: number; // char offset in the source string
end: number;
}
/**
* Segment in the visitor's language: sentence breaking differs by locale, and
* the generated text is in whatever language the prompt was written in.
* A browser reporting a malformed tag would make the constructor throw, so an
* unusable locale falls back to English rather than taking the app down.
*/
function makeSegmenter(): Intl.Segmenter | null {
if (typeof Intl === 'undefined' || !('Segmenter' in Intl)) return null;
for (const locale of [browserLocale(), 'en']) {
try {
return new Intl.Segmenter(locale, { granularity: 'sentence' });
} catch {
/* try the next one */
}
}
return null;
}
const segmenter = makeSegmenter();
export function splitSentences(text: string): SentenceSpan[] {
if (!text) return [];
const spans: SentenceSpan[] = [];
if (segmenter) {
for (const seg of segmenter.segment(text)) {
const trimmed = seg.segment.trim();
if (trimmed.length === 0) continue;
spans.push({ text: trimmed, start: seg.index, end: seg.index + seg.segment.length });
}
return spans;
}
// Regex fallback (node < 16 / very old browsers)
const re = /[^.!?]+[.!?]+(\s+|$)|[^.!?]+$/g;
let m: RegExpExecArray | null;
while ((m = re.exec(text)) !== null) {
const trimmed = m[0].trim();
if (trimmed.length > 0) spans.push({ text: trimmed, start: m.index, end: m.index + m[0].length });
}
return spans;
}
/**
* True if the text ends at a sentence boundary. Newlines count: UAX #29 (and
* therefore Intl.Segmenter, which the detector uses) treats paragraph
* separators as sentence breaks, so the generator must too — otherwise a
* markdown heading like "**Title**\n\n" would be fused with the next sentence
* at generation time but split at detection time, desynchronizing the
* k-SemStamp cluster chain.
*/
export function endsSentence(text: string): boolean {
return /[.!?]["')\]]?\s*$/.test(text) || /\n\s*$/.test(text);
}
|