tests.py

from unittest import TestCase

from tokenizers.pre_tokenizers import Whitespace

from recognizers.utils import DifferenceSample


class DifferenceSampleTestCase(TestCase):

    def setUp(self):
        self.text_a = "Chinese shares close higher Friday."
        self.text_b = "Les actions chinoises clôturent en baisse mercredi."
        self.tokenizer = Whitespace()
        self.encoding_a = self.tokenizer.pre_tokenize_str(self.text_a)
        self.encoding_b = self.tokenizer.pre_tokenize_str(self.text_b)
        self.result = DifferenceSample(
            tokens_a=tuple([token[0] for token in self.encoding_a]),
            tokens_b=tuple([token[0] for token in self.encoding_b]),
            labels_a=tuple([0.1 for _ in range(len(self.encoding_a))]),
            labels_b=tuple([0.1 for _ in range(len(self.encoding_b))]),
        )

    def test_add_whitespace(self):
        self.result.add_whitespace(self.encoding_a, self.encoding_b)
        self.assertEqual("".join(self.result.tokens_a), self.text_a)
        self.assertEqual("".join(self.result.tokens_b), self.text_b)