group singular and plural forms in clouds

handler.py

@@ -10,7 +10,7 @@ nltk.download('stopwords')
 
 from utils import clean_str, clean_str_nopunct
 import torch
-from utils import MultiHeadModel, BertInputBuilder, get_num_words, MATH_PREFIXES, MATH_WORDS
+from utils import MultiHeadModel, BertInputBuilder, get_num_words, MATH_PREFIXES, MATH_WORDS, plural_to_singular
 
 import transformers
 from transformers import BertTokenizer, BertForSequenceClassification
@@ -182,20 +182,20 @@ class Transcript:
                 teacher_dict.update(general_words)
             else:
                 student_dict.update(general_words)
-        dict_list = []
-        uptake_dict_list = []
-        teacher_dict_list = []
-        student_dict_list = []
-        for word in uptake_teacher_dict.keys():
-            uptake_dict_list.append({'text': word, 'value': uptake_teacher_dict[word], 'category': 'teacher'})
-        for word in teacher_dict.keys():
-            teacher_dict_list.append(
-                {'text': word, 'value': teacher_dict[word], 'category': 'general'})
-            dict_list.append({'text': word, 'value': teacher_dict[word], 'category': 'general'})
-        for word in student_dict.keys():
-            student_dict_list.append(
-                {'text': word, 'value': student_dict[word], 'category': 'general'})
-            dict_list.append({'text': word, 'value': student_dict[word], 'category': 'general'})
+
+        def dict_to_list(d, category):
+            combined_dict = Counter()
+            for word, count in d.items():
+                singular_word = plural_to_singular(word)
+                combined_dict[singular_word] += count
+            return [{'text': word, 'value': count, 'category': category} for word, count in combined_dict.items()]
+
+        # Sorting and trimming dictionaries
+        dict_list = dict_to_list(teacher_dict, 'general') + dict_to_list(student_dict, 'general')
+        uptake_dict_list = dict_to_list(uptake_teacher_dict, 'teacher')
+        teacher_dict_list = dict_to_list(teacher_dict, 'general')
+        student_dict_list = dict_to_list(student_dict, 'general')
+
         sorted_dict_list = sorted(dict_list, key=lambda x: x['value'], reverse=True)
         sorted_uptake_dict_list = sorted(uptake_dict_list, key=lambda x: x['value'], reverse=True)
         sorted_teacher_dict_list = sorted(teacher_dict_list, key=lambda x: x['value'], reverse=True)
@@ -265,7 +265,6 @@ class QuestionModel:
                             return_pooler_output=False)
         return output
 
-
 class ReasoningModel:
     def __init__(self, device, tokenizer, input_builder, max_length=128, path=REASONING_MODEL):
         print("Loading models...")
@@ -299,7 +298,6 @@ class ReasoningModel:
                             token_type_ids=instance["token_type_ids"])
         return output
 
-
 class UptakeModel:
     def __init__(self, device, tokenizer, input_builder, max_length=120, path=UPTAKE_MODEL):
         print("Loading models...")

utils.py

@@ -811,7 +811,11 @@ p = inflect.engine()
 def singular_to_plural(word):
     """Convert singular words to plural using inflect."""
     plural = p.plural(word)
-    return plural if plural else word  
+    return plural or word  
+
+def plural_to_singular(word):
+    """Convert plural word to singular using inflect."""
+    return p.singular_noun(word) or word 
 
 plural_MATH_WORDS = [singular_to_plural(word) for word in MATH_WORDS]