src/lib/utils/sentences.ts

/**
 * Sentence segmentation via Intl.Segmenter (replaces nltk sent_tokenize used
 * by the SemStamp reference implementation).
 */

import { browserLocale } from './locale';

export interface SentenceSpan {
  text: string;
  start: number; // char offset in the source string
  end: number;
}

/**
 * Segment in the visitor's language: sentence breaking differs by locale, and
 * the generated text is in whatever language the prompt was written in.
 * A browser reporting a malformed tag would make the constructor throw, so an
 * unusable locale falls back to English rather than taking the app down.
 */
function makeSegmenter(): Intl.Segmenter | null {
  if (typeof Intl === 'undefined' || !('Segmenter' in Intl)) return null;
  for (const locale of [browserLocale(), 'en']) {
    try {
      return new Intl.Segmenter(locale, { granularity: 'sentence' });
    } catch {
      /* try the next one */
    }
  }
  return null;
}

const segmenter = makeSegmenter();

export function splitSentences(text: string): SentenceSpan[] {
  if (!text) return [];
  const spans: SentenceSpan[] = [];
  if (segmenter) {
    for (const seg of segmenter.segment(text)) {
      const trimmed = seg.segment.trim();
      if (trimmed.length === 0) continue;
      spans.push({ text: trimmed, start: seg.index, end: seg.index + seg.segment.length });
    }
    return spans;
  }
  // Regex fallback (node < 16 / very old browsers)
  const re = /[^.!?]+[.!?]+(\s+|$)|[^.!?]+$/g;
  let m: RegExpExecArray | null;
  while ((m = re.exec(text)) !== null) {
    const trimmed = m[0].trim();
    if (trimmed.length > 0) spans.push({ text: trimmed, start: m.index, end: m.index + m[0].length });
  }
  return spans;
}

/**
 * True if the text ends at a sentence boundary. Newlines count: UAX #29 (and
 * therefore Intl.Segmenter, which the detector uses) treats paragraph
 * separators as sentence breaks, so the generator must too — otherwise a
 * markdown heading like "**Title**\n\n" would be fused with the next sentence
 * at generation time but split at detection time, desynchronizing the
 * k-SemStamp cluster chain.
 */
export function endsSentence(text: string): boolean {
  return /[.!?]["')\]]?\s*$/.test(text) || /\n\s*$/.test(text);
}