distinct

distinct.py

@@ -117,7 +117,6 @@ class distinct(evaluate.Measurement):
 
     def _compute(self, predictions, dataForVocabCal=None, vocab_size=None, tokenizer=Tokenizer13a(), mode="Expectation-Adjusted-Distinct"):
         from nltk.util import ngrams
-        from nltk.tokenize import WhitespaceTokenizer
 
         
         
@@ -128,9 +127,7 @@ class distinct(evaluate.Measurement):
             raise Warning("We've detected that both vocab_size and dataForVocabCal are specified. We will use dataForVocabCal.")
         elif mode == "Distinct":
             pass
-        
-        if tokenizer == "white_space":
-            tokenizer = WhitespaceTokenizer()
+    
         
         if mode == "Expectation-Adjusted-Distinct" and dataForVocabCal is not None:
             if isinstance(dataForVocabCal, list) and len(dataForVocabCal) > 0 and isinstance(dataForVocabCal[0], str):
@@ -152,10 +149,7 @@ class distinct(evaluate.Measurement):
 
         for prediction in predictions:          
             try:
-                print(prediction)
-                print(tokenizer.tokenize(prediction))
                 tokens = list(tokenizer.tokenize(prediction))
-                print(tokens)
                 tokens_2grams = list(ngrams(list(tokenizer.tokenize(prediction)), 2, pad_left=True, left_pad_symbol='<s>'))
                 tokens_3grams = list(ngrams(list(tokenizer.tokenize(prediction)), 3, pad_left=True, left_pad_symbol='<s>'))                   
             except Exception as e: