distinct

tokenizer_13a.py

@@ -67,8 +67,7 @@ class TokenizerRegexp(BaseTokenizer):
         # no leading or trailing spaces, single space within words
         # return ' '.join(line.split())
         # This line is changed with regards to the original tokenizer (seen above) to return individual words
-        print(line)
-        print("1:", line.split())
+
         return line.split()
 
 
@@ -98,7 +97,6 @@ class Tokenizer13a(BaseTokenizer):
             line = line.replace("&", "&")
             line = line.replace("&lt;", "<")
             line = line.replace("&gt;", ">")
-        print(line)
 
         return self._post_tokenizer(f" {line} ")