update distinct

distinct.py

@@ -146,27 +146,27 @@ class distinct(evaluate.Measurement):
         total_tokens = []
         total_tokens_2grams = []
         total_tokens_3grams = []
+
         for prediction in predictions:
             if tokenizer == "white_space":
                 tokens = prediction.split(" ")
-                tokens_2grams = ngrams(prediction.split(" "), 2, left_pad_symbol='<s>')
-                tokens_3grams = ngrams(prediction.split(" "), 3, left_pad_symbol='<s>')            
+                tokens_2grams = list(ngrams(prediction.split(" "), 2, pad_left=True, left_pad_symbol='<s>'))
+                tokens_3grams = list(ngrams(prediction.split(" "), 3, pad_left=True, left_pad_symbol='<s>'))           
             else:
                 try:
                     tokens = list(tokenizer.tokenize(prediction))
-                    tokens_2grams = ngrams(list(tokenizer.tokenize(prediction)), 2, left_pad_symbol='<s>')
-                    tokens_3grams = ngrams(list(tokenizer.tokenize(prediction)), 3, left_pad_symbol='<s>')
-                    
+                    tokens_2grams = list(ngrams(list(tokenizer.tokenize(prediction)), 2, pad_left=True, left_pad_symbol='<s>'))
+                    tokens_3grams = list(ngrams(list(tokenizer.tokenize(prediction)), 3, pad_left=True, left_pad_symbol='<s>'))                   
                 except Exception as e:
                     raise e
-            print(tokens_2grams)
+
             distinct_tokens = distinct_tokens | set(tokens)
             distinct_tokens_2grams = distinct_tokens_2grams | set(tokens_2grams)
             distinct_tokens_3grams = distinct_tokens_3grams | set(tokens_3grams)
             total_tokens.extend(tokens)
             total_tokens_2grams.extend(list(tokens_2grams))
             total_tokens_3grams.extend(list(tokens_3grams))
-        print(distinct_tokens_2grams, total_tokens_2grams)    
+    
         Distinct_1 = len(distinct_tokens)/len(total_tokens)
         Distinct_2 = len(distinct_tokens_2grams)/len(total_tokens_2grams)
         Distinct_3 = len(distinct_tokens_3grams)/len(total_tokens_3grams)