src/russian/parser.ts

// parser.ts — Russian UDPipe wrapper.
// Uses udpipe-node/wasm with the Russian SynTagRus model, producing RuTokens
// with UPOS tags, morphological features, and dependency relations.
//
// Unlike the English parser (src/parser.ts), this works with UPOS tags
// directly — the Russian UDPipe model does not produce XPOS (Penn Treebank).

import { createUDPipe } from 'udpipe-node/wasm';
import type { UDSentence } from 'udpipe-node';
import { fileURLToPath } from 'node:url';
import { dirname, resolve } from 'node:path';
import { existsSync } from 'node:fs';
import { CONTENT_UPOS, PUNCT_UPOS, type RuToken } from './types.js';

const __dirname = dirname(fileURLToPath(import.meta.url));

let _nlp: ReturnType<typeof createUDPipe> | null = null;

function findModel(): string | undefined {
  // 1. Explicit env var
  if (process.env.CALLIOPE_RUSSIAN_UDPIPE_MODEL) {
    return process.env.CALLIOPE_RUSSIAN_UDPIPE_MODEL;
  }
  // 2. Bundled alongside the package (data/ next to the compiled module is
  //    the canonical location; the RussianScan checkout paths cover dev
  //    layouts where this repo sits inside or beside the RussianScan clone).
  const candidates = [
    resolve(__dirname, 'data/russian-syntagrus-ud-2.0-170801.udpipe'),
    resolve(__dirname, '../../RussianScan/models/russian-syntagrus-ud-2.0-170801.udpipe'),
    resolve(__dirname, '../../../RussianScan/models/russian-syntagrus-ud-2.0-170801.udpipe'),
    resolve(__dirname, '../../models/russian-syntagrus-ud-2.0-170801.udpipe'),
  ];
  for (const p of candidates) {
    if (existsSync(p)) return p;
  }
  return undefined;
}

function nlp(): ReturnType<typeof createUDPipe> {
  if (_nlp) return _nlp;
  const modelPath = findModel();
  if (!modelPath) {
    // Without the SynTagRus model udpipe-node silently falls back to its
    // bundled ENGLISH model — every Russian token comes back NOUN/compound
    // and the whole pipeline (stress variants, ёфикация by gram, rhyme
    // feats, F&H maxima) degrades garbage-in.  Fail loudly instead.
    throw new Error(
      'Russian UDPipe model not found: put russian-syntagrus-ud-2.0-170801.udpipe ' +
      'in src/russian/data/ (and dist/russian/data/) or set CALLIOPE_RUSSIAN_UDPIPE_MODEL.'
    );
  }
  _nlp = createUDPipe({ defaultInputMode: 'tokenize', modelPath });
  return _nlp;
}

/** Pre-process text: space out punctuation so UDPipe doesn't glue it to words. */
function preprocessPunctuation(text: string): string {
  // Same approach as the original Python (poetry_alignment.py line 1264)
  const punctChars = '\'.‚,?!:;…-–—«»″""„‘’`ʹ"˝[]‹›·<>*/=()+®©‛¨×№\u05f4';
  let result = text;
  for (const c of punctChars) {
    result = result.split(c).join(' ' + c + ' ');
  }
  // Collapse multiple spaces
  result = result.replace(/  +/g, ' ');
  return result;
}

/** Parse Russian text into tokens with morphological features. */
export function parseRussianText(text: string): RuToken[][] {
  const preprocessed = preprocessPunctuation(text);
  const sentences = nlp().parse(preprocessed);
  return sentences.map((sent, si) => sent.words.map((w, wi) => ({
    form: w.form,
    lemma: w.lemma || w.form.toLowerCase(),
    upos: w.upos || 'X',
    feats: w.featsMap || {},
    deprel: w.deprel || '',
    head: w.head,
    id: (w as { id?: number }).id ?? wi + 1,
    sent: si,
    isContent: CONTENT_UPOS.has(w.upos || ''),
  })));
}

/** Check if a token is punctuation. */
export function isRuPunctuation(upos: string): boolean {
  return PUNCT_UPOS.has(upos);
}

/** Detect if text is predominantly Cyrillic (Russian). */
export function isRussianText(text: string): boolean {
  const cyrillic = (text.match(/[А-Яа-яЁё]/g) || []).length;
  const latin = (text.match(/[A-Za-z]/g) || []).length;
  return cyrillic > latin && cyrillic > 0;
}