Add model

config.json

@@ -0,0 +1,28 @@
+{
+  "_name_or_path": "roberta-base",
+  "architectures": [
+    "RobertaForSequenceClassification"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "bos_token_id": 0,
+  "classifier_dropout": null,
+  "eos_token_id": 2,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "layer_norm_eps": 1e-05,
+  "max_position_embeddings": 514,
+  "model_type": "roberta",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "pad_token_id": 1,
+  "position_embedding_type": "absolute",
+  "problem_type": "single_label_classification",
+  "torch_dtype": "float32",
+  "transformers_version": "4.28.0",
+  "type_vocab_size": 1,
+  "use_cache": true,
+  "vocab_size": 50265
+}

eval_results.txt

@@ -0,0 +1,12 @@
+accuracy = 1.0
+auprc = 1.0
+auroc = 1.0
+eval_loss = 5.9566273298529246e-05
+f1 = 1.0
+fn = 0
+fp = 0
+mcc = 1.0
+precision = 1.0
+recall = 1.0
+tn = 2262
+tp = 241

handler.py

@@ -0,0 +1,171 @@
+from simpletransformers.classification import ClassificationModel, ClassificationArgs
+from typing import Dict, List, Any
+import pandas as pd
+import webvtt
+from datetime import datetime
+import torch
+import spacy
+
+nlp = spacy.load("en_core_web_sm")
+tokenizer = nlp.tokenizer
+token_limit = 200
+
+class Utterance(object):
+
+    def __init__(self, starttime, endtime, speaker, text, 
+                 idx, prev_utterance, prev_prev_utterance):
+        self.starttime = starttime
+        self.endtime = endtime
+        self.speaker = speaker
+        self.text = text
+        self.idx = idx
+        self.prev_utterance = prev_utterance
+        self.prev_prev_utterance = prev_prev_utterance
+
+class EndpointHandler():
+    def __init__(self, path="."):
+        print("Loading models...")
+        cuda_available = torch.cuda.is_available()
+        self.model = ClassificationModel(
+            "roberta", path, use_cuda=cuda_available
+        )
+
+    def utterance_to_str(self, utterance: Utterance) -> (List[str], str):
+        #model utterance uses prior text
+        
+        doc = nlp(utterance.text)
+        prior_text = self.get_prior_text(utterance)
+
+        if len(doc) > token_limit:
+            utterance_text_list = self.handle_long_utterances(doc)
+            utterance_with_prior_text = []
+            for text in utterance_text_list:
+                utterance_with_prior_text.append([prior_text, text])
+            return utterance_with_prior_text, 'list'
+
+        else:
+            return [prior_text, utterance.text], 'single'
+
+    def format_speaker(self, speaker: str, source: str) -> str:
+        prior_text = ''
+        if speaker == 'student':
+            prior_text += '***STUDENT '
+        else:
+            prior_text += '***SECTION_LEADER '
+        if source == 'not chat':
+            prior_text += '(audio)*** : '
+        else:
+            prior_text += '(chat)*** : '
+        return prior_text
+    
+    def get_prior_text(self, utterance: Utterance) -> str:
+        prior_text = ''
+        if utterance.prev_utterance != None and utterance.prev_prev_utterance != None:
+            #TODO: add in the source
+            prior_text = '\"' + self.format_speaker(utterance.prev_prev_utterance.speaker, 'not chat') + utterance.prev_prev_utterance.text + ' \n ' 
+            prior_text += self.format_speaker(utterance.prev_utterance.speaker, 'not chat') + utterance.prev_utterance.text + ' \n '
+        else: 
+            prior_text = 'No prior utterance'
+        return prior_text
+    
+    def handle_long_utterances(self, doc: str) -> List[str]:
+        split_count = 1
+        total_sent = len([x for x in doc.sents])
+        sent_count = 0
+        token_count = 0
+        split_utterance = ''
+        utterances = []
+        for sent in doc.sents:
+            # add a sentence to split
+            split_utterance = split_utterance + ' ' + sent.text
+            token_count += len(sent)
+            sent_count +=1
+            if token_count >= token_limit or sent_count == total_sent: 
+                # save utterance segment
+                utterances.append(split_utterance)
+
+                # restart count
+                split_utterance = ''
+                token_count = 0
+                split_count += 1
+
+        return utterances
+    
+    def convert_time(self, time_str):
+        time = datetime.strptime(time_str, "%H:%M:%S.%f")
+        return 1000 * (3600 * time.hour + 60 * time.minute + time.second) + time.microsecond / 1000
+        
+    def process_vtt_transcript(self, vttfile) -> List[Utterance]:
+        """Process raw vtt file."""
+
+        utterances_list = []
+        text = ""
+        prev_start = "00:00:00.000"
+        prev_end = "00:00:00.000"
+        idx = 0
+        prev_speaker = None
+        prev_utterance = None
+        prev_prev_utterance = None
+        for caption in webvtt.read(vttfile):
+
+            # Get speaker
+            check_for_speaker = caption.text.split(":")
+            if len(check_for_speaker) > 1:  # the speaker was changed or restated
+                speaker = check_for_speaker[0]
+            else:
+                speaker = prev_speaker
+
+            # Get utterance
+            new_text = check_for_speaker[1] if len(check_for_speaker) > 1 else check_for_speaker[0]
+
+            # If speaker was changed, start new batch
+            if (prev_speaker is not None) and (speaker != prev_speaker):
+                utterance = Utterance(starttime=self.convert_time(prev_start),
+                              endtime=self.convert_time(prev_end),
+                              speaker=prev_speaker,
+                              text=text.strip(),
+                              idx=idx,
+                              prev_utterance=prev_utterance,
+                              prev_prev_utterance=prev_prev_utterance)
+
+                utterances_list.append(utterance)
+
+                # Start new batch
+                prev_start = caption.start
+                text = ""
+                prev_prev_utterance = prev_utterance
+                prev_utterance = utterance
+                idx+=1
+            text += new_text + " "
+            prev_end = caption.end
+            prev_speaker = speaker
+            
+        # Append last one
+        if prev_speaker is not None:
+            utterance = Utterance(starttime=self.convert_time(prev_start),
+                                  endtime=self.convert_time(prev_end),
+                                  speaker=prev_speaker,
+                                  text=text.strip(),
+                                  idx=idx,
+                                  prev_utterance=prev_utterance,
+                                  prev_prev_utterance=prev_prev_utterance)
+            utterances_list.append(utterance)
+
+        return utterances_list
+
+
+    def __call__(self, data_file: str) -> List[Dict[str, Any]]:
+        ''' data_file is a str pointing to filename of type .vtt '''
+
+        utterances_list = []
+        for utterance in self.process_vtt_transcript(data_file):
+            #TODO: filter out to only have SL utterances
+            utterance_str, is_list = self.utterance_to_str(utterance)
+            if is_list == 'list':
+                utterances_list.extend(utterance_str)
+            else:
+                utterances_list.append(utterance_str)
+                    
+        predictions, raw_outputs = self.model.predict(utterances_list)
+        
+        return predictions

model_args.json

@@ -0,0 +1 @@
+{"adafactor_beta1": null, "adafactor_clip_threshold": 1.0, "adafactor_decay_rate": -0.8, "adafactor_eps": [1e-30, 0.001], "adafactor_relative_step": true, "adafactor_scale_parameter": true, "adafactor_warmup_init": true, "adam_betas": [0.9, 0.999], "adam_epsilon": 1e-08, "best_model_dir": "outputs/roberta/model_utterance_FINAL_MODEL/best_model_all_transcripts", "cache_dir": "outputs/roberta/model_utterance_FINAL_MODEL/cache", "config": {}, "cosine_schedule_num_cycles": 0.5, "custom_layer_parameters": [], "custom_parameter_groups": [], "dataloader_num_workers": 0, "do_lower_case": false, "dynamic_quantize": false, "early_stopping_consider_epochs": false, "early_stopping_delta": 0, "early_stopping_metric": "eval_loss", "early_stopping_metric_minimize": true, "early_stopping_patience": 3, "encoding": null, "eval_batch_size": 8, "evaluate_during_training": true, "evaluate_during_training_silent": true, "evaluate_during_training_steps": 565, "evaluate_during_training_verbose": false, "evaluate_each_epoch": true, "fp16": false, "gradient_accumulation_steps": 2, "learning_rate": 4e-05, "local_rank": -1, "logging_steps": 50, "loss_type": null, "loss_args": {}, "manual_seed": null, "max_grad_norm": 1.0, "max_seq_length": 256, "model_name": "roberta-base", "model_type": "roberta", "multiprocessing_chunksize": -1, "n_gpu": 1, "no_cache": false, "no_save": false, "not_saved_args": [], "num_train_epochs": 5, "optimizer": "AdamW", "output_dir": "outputs/roberta/model_utterance_FINAL_MODEL", "overwrite_output_dir": true, "polynomial_decay_schedule_lr_end": 1e-07, "polynomial_decay_schedule_power": 1.0, "process_count": 1, "quantized_model": false, "reprocess_input_data": true, "save_best_model": true, "save_eval_checkpoints": false, "save_model_every_epoch": false, "save_optimizer_and_scheduler": true, "save_steps": 2000, "scheduler": "linear_schedule_with_warmup", "silent": false, "skip_special_tokens": true, "tensorboard_dir": "outputs/roberta/model_utterance_FINAL_MODEL/tensorboard", "thread_count": null, "tokenizer_name": "roberta-base", "tokenizer_type": null, "train_batch_size": 8, "train_custom_parameters_only": false, "use_cached_eval_features": false, "use_early_stopping": false, "use_hf_datasets": false, "use_multiprocessing": false, "use_multiprocessing_for_evaluation": false, "wandb_kwargs": {"reinit": true}, "wandb_project": "model_utterance_all_transcripts", "warmup_ratio": 0.06, "warmup_steps": 85, "weight_decay": 0.0, "model_class": "ClassificationModel", "labels_list": [0, 1], "labels_map": {}, "lazy_delimiter": "\t", "lazy_labels_column": 1, "lazy_loading": false, "lazy_loading_start_line": 1, "lazy_text_a_column": null, "lazy_text_b_column": null, "lazy_text_column": 0, "onnx": false, "regression": false, "sliding_window": false, "special_tokens_list": [], "stride": 0.8, "tie_value": 1}

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8ba1fcd9cd1ccf8636c01436ef0ba4a0b736244197e7936e2acc6be587d51197
+size 498662069

special_tokens_map.json

@@ -0,0 +1,15 @@
+{
+  "bos_token": "<s>",
+  "cls_token": "<s>",
+  "eos_token": "</s>",
+  "mask_token": {
+    "content": "<mask>",
+    "lstrip": true,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "<pad>",
+  "sep_token": "</s>",
+  "unk_token": "<unk>"
+}

test_run_handler.py

@@ -0,0 +1,13 @@
+from handler import EndpointHandler
+
+# init handler
+my_handler = EndpointHandler(path=".")
+
+# prepare sample payload
+test_payload = 'test.transcript.vtt'
+
+# test the handler
+test_pred=my_handler(test_payload)
+
+# show results
+print("test_pred", test_pred)

tokenizer_config.json

@@ -0,0 +1,16 @@
+{
+  "add_prefix_space": false,
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": true,
+  "cls_token": "<s>",
+  "do_lower_case": false,
+  "eos_token": "</s>",
+  "errors": "replace",
+  "mask_token": "<mask>",
+  "model_max_length": 512,
+  "pad_token": "<pad>",
+  "sep_token": "</s>",
+  "tokenizer_class": "RobertaTokenizer",
+  "trim_offsets": true,
+  "unk_token": "<unk>"
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:90c1b64e7276b2247df250b6e1c458a2e5b2ce6dfad37780d09a9761df9499ee
+size 3451

training_progress_scores.csv

@@ -0,0 +1,8 @@
+global_step,train_loss,mcc,tp,tn,fp,fn,auroc,auprc,accuracy,precision,recall,f1,eval_loss
+283,0.3241749703884125,0.8551549806677315,228,2205,57,13,0.9911931203246127,0.9301238510748596,0.9720335597283261,0.946058091286307,0.8,0.8669201520912547,0.07226471802533661
+565,0.0068567488342523575,0.9461676303820653,240,2238,24,1,0.9994827035891566,0.9972630809896331,0.9900119856172593,0.995850622406639,0.9090909090909091,0.9504950495049505,0.03605068988365176
+566,0.00029088457813486457,0.9172646185925842,240,2223,39,1,0.9993634686008416,0.9968337615467682,0.9840191769876149,0.995850622406639,0.8602150537634409,0.923076923076923,0.05396954482425928
+849,0.00020018930081278086,1.0,241,2262,0,0,1.0,1.0,1.0,1.0,1.0,1.0,0.0004376148809996856
+1130,7.240189734147862e-05,1.0,241,2262,0,0,1.0,1.0,1.0,1.0,1.0,1.0,0.00017808274872814522
+1132,6.916876009199768e-05,1.0,241,2262,0,0,1.0,1.0,1.0,1.0,1.0,1.0,0.0001942521630669758
+1415,6.425171159207821e-05,1.0,241,2262,0,0,1.0,1.0,1.0,1.0,1.0,1.0,5.9566273298529246e-05