reverting

handler.py

@@ -3,9 +3,6 @@ from scipy.special import softmax
 import numpy as np
 import weakref
 import re
-import nltk
-from nltk.corpus import stopwords
-nltk.download('stopwords')
 
 from utils import clean_str, clean_str_nopunct
 import torch
@@ -13,7 +10,7 @@ from utils import MultiHeadModel, BertInputBuilder, get_num_words, MATH_PREFIXES
 
 import transformers
 from transformers import BertTokenizer, BertForSequenceClassification
-from transformers.utils import logging
+
 
 transformers.logging.set_verbosity_debug()
 
@@ -33,15 +30,9 @@ class Utterance:
         self.endtime = endtime
         self.transcript = weakref.ref(transcript) if transcript else None
         self.props = kwargs
-        self.role = None
-        self.word_count = self.get_num_words()
-        self.timestamp = [starttime, endtime]
-        self.unit_measure = None
-        self.aggregate_unit_measure = endtime
         self.num_math_terms = None
         self.math_terms = None
 
-        # moments
         self.uptake = None
         self.reasoning = None
         self.question = None
@@ -71,20 +62,6 @@ class Utterance:
             **self.props
         }
 
-    def to_talk_timeline_dict(self):
-        return{
-            'speaker': self.speaker,
-            'text': self.text,
-            'role': self.role,
-            'timestamp': self.timestamp,
-            'moments': {'reasoning': True if self.reasoning else False, 'questioning': True if self.question else False, 'uptake': True if self.uptake else False, 'focusingQuestion': True if self.focusing_question else False},
-            'unitMeasure': self.unit_measure,
-            'aggregateUnitMeasure': self.aggregate_unit_measure,
-            'wordCount': self.word_count,
-            'numMathTerms': self.num_math_terms,
-            'mathTerms': self.math_terms
-        }
-
     def __repr__(self):
         return f"Utterance(speaker='{self.speaker}'," \
                f"text='{self.text}', uid={self.uid}," \
@@ -114,86 +91,6 @@ class Transcript:
     def length(self):
         return len(self.utterances)
 
-    def update_utterance_roles(self, uptake_speaker):
-        for utt in self.utterances:
-            if (utt.speaker == uptake_speaker):
-                utt.role = 'teacher'
-            else:
-                utt.role = 'student'
-
-    def get_talk_distribution_and_length(self, uptake_speaker):
-        if ((uptake_speaker is None)):
-            return None
-        teacher_words = 0
-        teacher_utt_count = 0
-        student_words = 0
-        student_utt_count = 0
-        for utt in self.utterances:
-            if (utt.speaker == uptake_speaker):
-                utt.role = 'teacher'
-                teacher_words += utt.get_num_words()
-                teacher_utt_count += 1
-            else:
-                utt.role = 'student'
-                student_words += utt.get_num_words()
-                student_utt_count += 1
-        teacher_percentage = round(
-            (teacher_words / (teacher_words + student_words)) * 100)
-        student_percentage = 100 - teacher_percentage
-        avg_teacher_length = teacher_words / teacher_utt_count
-        avg_student_length = student_words / student_utt_count
-        return {'teacher': teacher_percentage, 'student': student_percentage}, {'teacher': avg_teacher_length, 'student': avg_student_length}
-
-    def get_word_cloud_dicts(self):
-        teacher_dict = {}
-        student_dict = {}
-        uptake_teacher_dict = {}
-        stop_words = stopwords.words('english')
-        # stopwords = nltk.corpus.stopwords.word('english')
-        # print("stopwords: ", stopwords)
-        for utt in self.utterances:
-            words = (utt.get_clean_text(remove_punct=True)).split(' ')
-            for word in words:
-                if word in stop_words: continue
-                if utt.role == 'teacher':
-                    if word not in teacher_dict:
-                        teacher_dict[word] = 0
-                    teacher_dict[word] += 1
-                    if utt.uptake == 1:
-                        if word not in uptake_teacher_dict:
-                            uptake_teacher_dict[word] = 0
-                        uptake_teacher_dict[word] += 1
-                else:
-                    if word not in student_dict:
-                        student_dict[word] = 0
-                    student_dict[word] += 1
-        dict_list = []
-        uptake_dict_list = []
-        for word in uptake_teacher_dict.keys():
-            uptake_dict_list.append({'text': word, 'value': uptake_teacher_dict[word], 'category': 'teacher'})
-        for word in teacher_dict.keys():
-            dict_list.append(
-                {'text': word, 'value': teacher_dict[word], 'category': 'teacher'})
-        for word in student_dict.keys():
-            dict_list.append(
-                {'text': word, 'value': student_dict[word], 'category': 'student'})
-        sorted_dict_list = sorted(dict_list, key=lambda x: x['value'], reverse=True)
-        sorted_uptake_dict_list = sorted(uptake_dict_list, key=lambda x: x['value'], reverse=True)
-        return sorted_dict_list[:50], sorted_uptake_dict_list[:50]
-
-    def get_talk_timeline(self):
-        return [utterance.to_talk_timeline_dict() for utterance in self.utterances]
-    
-    def calculate_aggregate_word_count(self):
-        unit_measures = [utt.unit_measure for utt in self.utterances]
-        if None in unit_measures:
-            aggregate_word_count = 0
-            for utt in self.utterances: 
-                aggregate_word_count += utt.get_num_words()
-                utt.unit_measure = utt.get_num_words()
-                utt.aggregate_unit_measure = aggregate_word_count
-
-
     def to_dict(self):
         return {
             'utterances': [utterance.to_dict() for utterance in self.utterances],
@@ -321,6 +218,8 @@ class UptakeModel:
                             return_pooler_output=False)
         return output
 
+
+
 class FocusingQuestionModel:
     def __init__(self, device, tokenizer, input_builder, max_length=128, path=FOCUSING_QUESTION_MODEL):
         print("Loading models...")
@@ -355,7 +254,8 @@ class FocusingQuestionModel:
         output = self.model(input_ids=instance["input_ids"],
                             attention_mask=instance["attention_mask"],
                             token_type_ids=instance["token_type_ids"])
-        return output   
+        return output        
+
 
 def load_math_terms():
     math_terms = []
@@ -365,29 +265,23 @@ def load_math_terms():
             math_terms_dict[f"(^|[^a-zA-Z]){term}(s|es)?([^a-zA-Z]|$)"] = term
             math_terms.append(f"(^|[^a-zA-Z]){term}(s|es)?([^a-zA-Z]|$)")
         else:
-            math_terms.append(term)
-            math_terms_dict[term] = term
+            math_terms_dict[f"(^|[^a-zA-Z]){term}([^a-zA-Z]|$)"] = term
+            math_terms.append(f"(^|[^a-zA-Z]){term}([^a-zA-Z]|$)")
     return math_terms, math_terms_dict
 
 def run_math_density(transcript):
     math_terms, math_terms_dict = load_math_terms()
-    sorted_terms = sorted(math_terms, key=len, reverse=True)
-    for i, utt in enumerate(transcript.utterances):
+    for i, utt in enumerate(transcript.utterances): 
+        found_math_terms = set()
         text = utt.get_clean_text(remove_punct=False)
-        num_matches = 0
-        matched_positions = set()
-        match_list = []
-        for term in sorted_terms:
-            matches = list(re.finditer(term, text, re.IGNORECASE))
-            # Filter out matches that share positions with longer terms
-            matches = [match for match in matches if not any(match.start() in range(existing[0], existing[1]) for existing in matched_positions)]
-            if len(matches) > 0:
-                match_list.append(math_terms_dict[term])
-            # Update matched positions
-            matched_positions.update((match.start(), match.end()) for match in matches)
-            num_matches += len(matches)
-        utt.num_math_terms = num_matches
-        utt.math_terms = match_list
+        num_math_terms = 0
+        for term in math_terms:
+            count = len(re.findall(term, text))
+            if count > 0:
+                found_math_terms.add(math_terms_dict[term])
+            num_math_terms += count
+        utt.num_math_terms = num_math_terms
+        utt.math_terms = list(found_math_terms)
 
 class EndpointHandler():
     def __init__(self, path="."):
@@ -419,13 +313,13 @@ class EndpointHandler():
             transcript.add_utterance(Utterance(**utt))
 
         print("Running inference on %d examples..." % transcript.length())
-        logging.set_verbosity_info()
+        uptake_speaker = params.pop("uptake_speaker", None)
         # Uptake
         uptake_model = UptakeModel(
             self.device, self.tokenizer, self.input_builder)
-        uptake_speaker = params.pop("uptake_speaker", None)
         uptake_model.run_inference(transcript, min_prev_words=params['uptake_min_num_words'],
                                    uptake_speaker=uptake_speaker)
+
         # Reasoning
         reasoning_model = ReasoningModel(
             self.device, self.tokenizer, self.input_builder)
@@ -443,17 +337,4 @@ class EndpointHandler():
 
         run_math_density(transcript)
 
-        transcript.update_utterance_roles(uptake_speaker)
-        transcript.calculate_aggregate_word_count()
-        return_dict = {'talkDistribution': None, 'talkLength': None, 'talkMoments': None, 'commonTopWords': None, 'uptakeTopWords': None}
-        talk_dist, talk_len = transcript.get_talk_distribution_and_length(uptake_speaker)
-        return_dict['talkDistribution'] = talk_dist
-        return_dict['talkLength'] = talk_len
-        talk_moments = transcript.get_talk_timeline()
-        return_dict['talkMoments'] = talk_moments
-        word_cloud, uptake_word_cloud = transcript.get_word_cloud_dicts()
-        return_dict['commonTopWords'] = word_cloud
-        return_dict['uptakeTopwords'] = uptake_word_cloud
-
-
-        return return_dict
+        return transcript.to_dict()