preprocessors/ssplit.py

# Moses style sentence splitter

import re
import typing as tp
from pathlib import Path

from sentence_splitter import SentenceSplitter

LANG_CODES = {
    "fra_Latn": "fr",
    "eng_Latn": "en"
}

def sent_splitter(text, lang_code) -> tp.Callable:
    "Split text"
    lang = LANG_CODES.get(lang_code, "en") # Default to English if code not present

    splitter = SentenceSplitter(language=lang)
    return splitter.split(text)
    
def test_splitter():
    fr_text = "Les rues de la ville s'éveillaient doucement sous la lumière dorée du matin. Les cafés ouvraient leurs terrasses, tandis que le bruit des pas résonnait sur les pavés humides. C'était un moment de calme avant l'agitation habituelle de la journée, où chacun semblait suivre son propre rythme, perdu dans ses pensées ou pressé par le temps."
    en_text = "The old library stood at the end of the cobblestone street, its brick walls covered in climbing ivy that had turned a deep crimson with the autumn chill. Inside, the air smelled faintly of dust and aged paper, a comforting reminder of countless stories waiting to be discovered. Every shelf seemed to hold a different world, and the quiet hum of turning pages was the only sound that dared to break the stillness."
    ig_text = "Obodo Igbo na-ebuli elu site n'ịdị n'otu na ịhụnanya. N'oge owuwe ihe ubi, ụmụ nwoke na ụmụ nwanyị na-arụkọ ọrụ ọnụ iji kpokọta ihe ọkụkụ ha. Ndị agadi na-akọ akụkọ oge ochie n'okpuru osisi ogilisi, ebe ụmụaka na-agba egwu ma na-akụ ọgba. Ihe a niile na-egosi na ezi obodo bụ nke mmadụ na ibe ya na-eme ihe ọnụ."

    en_sents = ["The old library stood at the end of the cobblestone street, its brick walls covered in climbing ivy that had turned a deep crimson with the autumn chill.",
      "Inside, the air smelled faintly of dust and aged paper, a comforting reminder of countless stories waiting to be discovered.",
      "Every shelf seemed to hold a different world, and the quiet hum of turning pages was the only sound that dared to break the stillness."
    ]
    fr_sents = ["Les rues de la ville s'éveillaient doucement sous la lumière dorée du matin.",
    "Les cafés ouvraient leurs terrasses, tandis que le bruit des pas résonnait sur les pavés humides.",
    "C'était un moment de calme avant l'agitation habituelle de la journée, où chacun semblait suivre son propre rythme, perdu dans ses pensées ou pressé par le temps."
    ]
    ig_sents = ["Obodo Igbo na-ebuli elu site n'ịdị n'otu na ịhụnanya.",
    "N'oge owuwe ihe ubi, ụmụ nwoke na ụmụ nwanyị na-arụkọ ọrụ ọnụ iji kpokọta ihe ọkụkụ ha.",
    "Ndị agadi na-akọ akụkọ oge ochie n'okpuru osisi ogilisi, ebe ụmụaka na-agba egwu ma na-akụ ọgba.",
    "Ihe a niile na-egosi na ezi obodo bụ nke mmadụ na ibe ya na-eme ihe ọnụ."
    ]
    
    assert(sent_splitter(en_text, "eng_Latn") == en_sents)
    assert (sent_splitter(fr_text, "fra_Latn") == fr_sents)
    assert(sent_splitter(ig_text, "ibo_Latn") == ig_sents)

# test_splitter()