// parser.ts — Russian UDPipe wrapper. // Uses udpipe-node/wasm with the Russian SynTagRus model, producing RuTokens // with UPOS tags, morphological features, and dependency relations. // // Unlike the English parser (src/parser.ts), this works with UPOS tags // directly — the Russian UDPipe model does not produce XPOS (Penn Treebank). import { createUDPipe } from 'udpipe-node/wasm'; import type { UDSentence } from 'udpipe-node'; import { fileURLToPath } from 'node:url'; import { dirname, resolve } from 'node:path'; import { existsSync } from 'node:fs'; import { CONTENT_UPOS, PUNCT_UPOS, type RuToken } from './types.js'; const __dirname = dirname(fileURLToPath(import.meta.url)); let _nlp: ReturnType | null = null; function findModel(): string | undefined { // 1. Explicit env var if (process.env.CALLIOPE_RUSSIAN_UDPIPE_MODEL) { return process.env.CALLIOPE_RUSSIAN_UDPIPE_MODEL; } // 2. Bundled alongside the package (data/ next to the compiled module is // the canonical location; the RussianScan checkout paths cover dev // layouts where this repo sits inside or beside the RussianScan clone). const candidates = [ resolve(__dirname, 'data/russian-syntagrus-ud-2.0-170801.udpipe'), resolve(__dirname, '../../RussianScan/models/russian-syntagrus-ud-2.0-170801.udpipe'), resolve(__dirname, '../../../RussianScan/models/russian-syntagrus-ud-2.0-170801.udpipe'), resolve(__dirname, '../../models/russian-syntagrus-ud-2.0-170801.udpipe'), ]; for (const p of candidates) { if (existsSync(p)) return p; } return undefined; } function nlp(): ReturnType { if (_nlp) return _nlp; const modelPath = findModel(); if (!modelPath) { // Without the SynTagRus model udpipe-node silently falls back to its // bundled ENGLISH model — every Russian token comes back NOUN/compound // and the whole pipeline (stress variants, ёфикация by gram, rhyme // feats, F&H maxima) degrades garbage-in. Fail loudly instead. throw new Error( 'Russian UDPipe model not found: put russian-syntagrus-ud-2.0-170801.udpipe ' + 'in src/russian/data/ (and dist/russian/data/) or set CALLIOPE_RUSSIAN_UDPIPE_MODEL.' ); } _nlp = createUDPipe({ defaultInputMode: 'tokenize', modelPath }); return _nlp; } /** Pre-process text: space out punctuation so UDPipe doesn't glue it to words. */ function preprocessPunctuation(text: string): string { // Same approach as the original Python (poetry_alignment.py line 1264) const punctChars = '\'.‚,?!:;…-–—«»″""„‘’`ʹ"˝[]‹›·<>*/=()+®©‛¨×№\u05f4'; let result = text; for (const c of punctChars) { result = result.split(c).join(' ' + c + ' '); } // Collapse multiple spaces result = result.replace(/ +/g, ' '); return result; } /** Parse Russian text into tokens with morphological features. */ export function parseRussianText(text: string): RuToken[][] { const preprocessed = preprocessPunctuation(text); const sentences = nlp().parse(preprocessed); return sentences.map((sent, si) => sent.words.map((w, wi) => ({ form: w.form, lemma: w.lemma || w.form.toLowerCase(), upos: w.upos || 'X', feats: w.featsMap || {}, deprel: w.deprel || '', head: w.head, id: (w as { id?: number }).id ?? wi + 1, sent: si, isContent: CONTENT_UPOS.has(w.upos || ''), }))); } /** Check if a token is punctuation. */ export function isRuPunctuation(upos: string): boolean { return PUNCT_UPOS.has(upos); } /** Detect if text is predominantly Cyrillic (Russian). */ export function isRussianText(text: string): boolean { const cyrillic = (text.match(/[А-Яа-яЁё]/g) || []).length; const latin = (text.match(/[A-Za-z]/g) || []).length; return cyrillic > latin && cyrillic > 0; }