libs/normalizer.py

import re
import regex
import sys
import textwrap
from typing import Any, Dict, Optional

punctuations = [
    '!', '"', '#', '$', '%', '&', "'", '(', ')', '*', '+', ',', '.',
    '/', ':', ';', '<', '=', '>', '?', '@', '[', '\\', ']', '^', '_',
    '`', '{', '|', '}', '~', '»', '«', '“', '”', "-",
]


class Normalizer:
    """A general normalizer for every language"""

    _whitelist = r"[" + "\p{N}\p{L}\p{M}" + re.escape("".join(punctuations)) + "]+"
    _dictionary = {}

    def __init__(
            self,
            whitelist: str = None,
            dictionary: Dict[str, str] = None,
    ) -> None:
        self.whitelist = whitelist if whitelist and isinstance(whitelist, str) else self._whitelist
        self.dictionary = dictionary if dictionary and isinstance(dictionary, dict) else self._dictionary

    def chars_to_map(self, sentence: str) -> str:
        """Maps every character, words, and phrase into a proper one.

        Args:
            sentence (str): A piece of text.
        """
        if not len(self.dictionary) > 0:
            return sentence

        pattern = "|".join(map(re.escape, self.dictionary.keys()))
        return re.sub(pattern, lambda m: self.dictionary[m.group()], str(sentence))

    def chars_to_preserve(
            self,
            sentence: str,
    ) -> str:
        """Keeps specified characters from sentence

        Args:
            sentence (str): A piece of text.
        """
        try:
            tokenized = regex.findall(self.whitelist, sentence)
            return " ".join(tokenized)
        except Exception as error:
            print(
                textwrap.dedent(
                    f"""
                    Bad characters range {self.whitelist},
                    {error}
                    """
                )
            )
            raise

    def text_level_normalizer(self, text: str) -> str:
        """A text level of normalization"""

        text = regex.sub(r"([" + re.escape("".join(punctuations)) + "])", r" \1 ", text)
        text = text.strip()

        return text

    def __call__(
            self,
            text: str,
            do_lowercase: Optional[bool] = False
    ) -> Any:
        """Normalization caller"""

        text = self.chars_to_map(text)
        text = self.chars_to_preserve(text)
        text = self.text_level_normalizer(text)
        text = re.sub(r"\s+", " ", text)

        if do_lowercase:
            text = text.lower()

        return text