preprocessors/tstrip.py

# Copyright (c) Meta Platforms, Inc. and affiliates.
# All rights reserved.
#
# This source code is licensed under the MIT License.

# Retrieved from https://github.com/hobodrifterdavid/nllb-docker-rest/tree/main/stopes_snippet
#
# This is supposed to be a drop in replacement to moses strip-non-printing-char.perl
import re
import sys
import typing as tp
import unicodedata


def regex_replacer(
    regex_pattern: str, replace_by: str = " ", flags: int = re.IGNORECASE
) -> tp.Callable[[str], str]:
    prog = re.compile(regex_pattern, flags=flags)

    def replacer(line) -> str:
        return prog.sub(replace_by, line)

    return replacer


def get_url_replacer(replace_by: str = " ") -> tp.Callable[[str], str]:
    url_pattern = r"http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+"
    return regex_replacer(url_pattern, replace_by)

def get_ascii_hashtag_replacer(replace_by: str = " ") -> tp.Callable[[str], str]:
    hashtag_pattern = r"[#@](\w\w\w\w+)"
    return regex_replacer(hashtag_pattern, replace_by, flags=(re.IGNORECASE | re.ASCII))

def get_non_printing_char_replacer(replace_by: str = " ") -> tp.Callable[[str], str]:
    non_printable_map = {
        ord(c): replace_by
        for c in (chr(i) for i in range(sys.maxunicode + 1))
        # same as \p{C} in perl
        # see https://www.unicode.org/reports/tr44/#General_Category_Values
        if unicodedata.category(c) in {"C", "Cc", "Cf", "Cs", "Co", "Cn"}
    }

    # Cleaning non-printable characters redundant given use of MosesPunctNormalizer?
    def replace_non_printing_char(line) -> str:
        return line.translate(non_printable_map)

    return replace_non_printing_char

def trim(text: str) -> str:
    """Take first 500 characters alone"""
    return text[:500]