Upload 2 files

.gitattributes

@@ -31,3 +31,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+BERTNN_model filter=lfs diff=lfs merge=lfs -text

BERTNN_model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3234a877d983726862db4eebe40e200ae6752c23302c0be8a20e47b5a0a0c412
+size 1341231081

predefined_bertnn.py

@@ -0,0 +1,716 @@
+
+## Import required libraries
+from datetime import datetime
+import numpy as np
+import pandas as pd
+import random
+from transformers import BertTokenizer, BertModel
+import logging
+import matplotlib.pyplot as plt
+tokenizer = BertTokenizer.from_pretrained('bert-large-uncased')#bert-large-uncased
+import itertools
+from sklearn.preprocessing import StandardScaler
+from itertools import cycle,islice
+from random import sample
+import torch
+import torch.nn as nn
+from torch.utils.data import DataLoader, RandomSampler, SequentialSampler
+import torch.nn.functional as F
+
+## ******  We use GPU if available   ******** ##
+if torch.cuda.is_available():       
+    device = torch.device("cuda")
+    print(f'There are {torch.cuda.device_count()} GPU(s) available.')
+    print('Device name:', torch.cuda.get_device_name(0))
+else:
+    print('No GPU available, using the CPU instead.')
+    device = torch.device("cpu")
+## ****** Initialize random seeds  ******** ##
+rnd_st=42
+np.random.seed(rnd_st)
+random.seed(rnd_st)
+torch.manual_seed(rnd_st)
+torch.cuda.manual_seed(rnd_st)
+# Running on the CuDNN backend
+torch.backends.cudnn.deterministic = True
+torch.backends.cudnn.benchmark = False
+
+## ****** Load and normalize the refrence EPA dictionary  ******** ## 
+def load_dictionary(file):
+    df=pd.read_csv(file).reset_index().rename(columns={"index": 'index_in_dic'})
+    df['term2']=df['term']
+    df.term=df.term.str.replace("_", " ") 
+    df['len_Bert']=df.apply(lambda x: len(tokenizer.tokenize(x['term'])),axis=1)
+    # df=add_cluster(df)
+    return(df)
+
+Modifiers =load_dictionary("FullSurveyorInteract_Modifiers.csv")
+# Behaviors=load_dictionary("FullSurveyorInteract_Behaviors.csv")
+Behaviors=load_dictionary("FullSurveyorInteract_Behaviors_3rd.csv")
+Identities=load_dictionary("FullSurveyorInteract_Identities.csv")
+
+n_Modifiers = Modifiers.copy()
+n_Behaviors =Behaviors.copy()
+n_Identities = Identities.copy()
+
+scaler_B,scaler_M,scaler_I = StandardScaler(),StandardScaler(),StandardScaler()
+
+n_Behaviors[['E','P','A']] = scaler_B.fit_transform(Behaviors[['E','P','A']])
+n_Modifiers[['E','P','A']] = scaler_M.fit_transform(Modifiers[['E','P','A']])
+n_Identities[['E','P','A']] = scaler_I.fit_transform(Identities[['E','P','A']])
+
+
+# Ref: https://mccormickml.com/2019/05/14/BERT-word-embeddings-tutorial/
+
+rnd_st=42
+
+# Load the BERT tokenizer
+tokenizer = BertTokenizer.from_pretrained('bert-large-uncased', do_lower_case=True)
+
+# Create a function to tokenize a set of texts
+def preprocessing_for_bert(data,MAX_LEN=40):
+    """Perform required preprocessing steps for pretrained BERT.
+    @param    data (np.array): Array of texts to be processed.
+    @return   input_ids (torch.Tensor): Tensor of token ids to be fed to a model.
+    @return   attention_masks (torch.Tensor): Tensor of indices specifying which
+                  tokens should be attended to by the model.
+    """
+    # Create empty lists to store outputs
+    input_ids = []
+    attention_masks = []
+
+    # For every sentence...
+    for sent in data:
+        # `encode_plus` will:
+        #    (1) Tokenize the sentence
+        #    (2) Add the `[CLS]` and `[SEP]` token to the start and end
+        #    (3) Truncate/Pad sentence to max length
+        #    (4) Map tokens to their IDs
+        #    (5) Create attention mask
+        #    (6) Return a dictionary of outputs
+        encoded_sent = tokenizer.encode_plus(
+            text=sent,  # Preprocess sentence
+            add_special_tokens=True,        # Add `[CLS]` and `[SEP]`
+            max_length=MAX_LEN,                  # Max length to truncate/pad
+            padding='max_length',
+            return_attention_mask=True      # Return attention mask
+            )
+        
+        # Add the outputs to the lists
+        input_ids.append(encoded_sent.get('input_ids'))
+        attention_masks.append(encoded_sent.get('attention_mask'))
+
+    # Convert lists to tensors
+    input_ids = torch.tensor(input_ids[0])
+    attention_masks = torch.tensor(attention_masks[0])
+
+    return input_ids, attention_masks
+
+
+# # Convert other data types to torch.Tensor
+# from torch.utils.data import TensorDataset, DataLoader, RandomSampler, SequentialSampler
+def gnrtr2(Identity,Behavior,Modifier):
+        ident1,ident2,behav=Identity.sample(axis = 0),Identity.sample(axis = 0),Behavior.sample(axis = 0)
+        modif1,modif2=Modifier.sample(axis = 0),Modifier.sample(axis = 0)
+        id1,id2,beh,mod1,mod2=list(ident1.term),list(ident2.term),list(behav.term),list(modif1.term),list(modif2.term)
+        sents=' '.join(map(str, (mod1+id1+beh+mod2+id2)))
+        values=np.concatenate([(modif1[['E','P','A']]).to_numpy(),
+                               (ident1[['E','P','A']]).to_numpy(),
+                              (behav[['E','P','A']]).to_numpy(),
+                               (modif2[['E','P','A']]).to_numpy(),
+                               (ident2[['E','P','A']]).to_numpy()], axis=1)[0]
+        indexx=torch.tensor([[(modif1['index_in_dic']).to_numpy()][0][0],
+                            [(ident1['index_in_dic']).to_numpy()][0][0],
+                            [(behav['index_in_dic']).to_numpy()][0][0],
+                            [(modif2['index_in_dic']).to_numpy()][0][0],
+                            [(ident2['index_in_dic']).to_numpy()][0][0]])
+        ys= torch.tensor(values)
+        inputs, masks = preprocessing_for_bert([sents])      
+        yield inputs, masks, ys,indexx  #torch.tensor(sents),
+
+# For fine-tuning BERT, the authors recommend a batch size of 16 or 32.
+def dta_ldr2(I,B,M,batch_size=32):
+    dt_ldr= [x for x in DataLoader([next(gnrtr2(I,B,M)) for x in range(batch_size)],  batch_size=batch_size)][0]
+    return(dt_ldr)
+
+
+
+# # Convert other data types to torch.Tensor
+
+# For fine-tuning BERT, the authors recommend a batch size of 16 or 32.
+def dta_ldr(I,B,M,batch_size=32):
+    dt_ldr= [x for x in DataLoader([next(gnrtr(I,B,M)) for x in range(batch_size)],  batch_size=batch_size)][0]
+    return(dt_ldr)
+class BertRegressor(nn.Module):
+    """Bert Model for Regression Tasks.
+    """
+    def __init__(self, freeze_bert=False):
+        """
+        @param    bert: a BertModel object
+        @param    classifier: a torch.nn.Module regressor
+        @param    freeze_bert (bool): Set `False` to fine-tune the BERT model
+        """
+        super(BertRegressor, self).__init__()
+        # Specify hidden size of BERT, hidden size of our regressor, and number of independent variables
+        D_in, H, D_out = 1024, 120, 15
+
+        # Instantiate BERT model
+        self.bert = BertModel.from_pretrained('bert-large-uncased')
+
+        # Instantiate an one-layer feed-forward classifier
+        self.regressor = nn.Sequential(
+            nn.Dropout(0.4),
+            nn.Linear(D_in, H),
+            nn.Dropout(0.3),
+            nn.ReLU(),
+            nn.Dropout(0.3),
+            nn.ReLU(),
+            nn.Dropout(0.3),
+            nn.Linear(H, D_out)
+        )
+
+        # Freeze the BERT model
+        if freeze_bert:
+            for param in self.bert.parameters():
+                param.requires_grad = False
+        
+    def forward(self, input_ids, attention_mask):
+        """
+        Feed input to BERT and the classifier to compute logits.
+        @param    input_ids (torch.Tensor): an input tensor with shape (batch_size,
+                      max_length)
+        @param    attention_mask (torch.Tensor): a tensor that hold attention mask
+                      information with shape (batch_size, max_length)
+        @return   logits (torch.Tensor): an output tensor with shape (batch_size,
+                      num_labels)
+        """
+        # Feed input to BERT
+        outputs = self.bert(input_ids=input_ids,
+                            attention_mask=attention_mask)
+        
+        # Extract the last hidden state of the token `[CLS]` for regression task
+        last_hidden_state_cls = outputs.pooler_output#outputs[0][:, 0, :]
+
+        # Feed input to classifier to compute predictions
+        predictions = self.regressor(last_hidden_state_cls)#.float()
+        return predictions#.float()
+from transformers import AdamW, get_linear_schedule_with_warmup
+
+def initialize_model(epochs=4):
+    """Initialize the Bert Classifier, the optimizer and the learning rate scheduler.
+    """
+    # Instantiate Bert Classifier
+    bert_regressor = BertRegressor(freeze_bert=False)
+
+    # Tell PyTorch to run the model on GPU
+    bert_regressor.to(device)
+
+    # Create the optimizer
+    optimizer = AdamW(bert_regressor.parameters(),
+                      lr=2e-5,    # Smaller LR
+                      eps=1e-8,    # Default epsilon value
+                      weight_decay =0.001  # Decoupled weight decay to apply.
+                      )
+
+    # Total number of training steps
+    total_steps = 100000#len(train_dataloader) * epochs
+
+    # Set up the learning rate scheduler
+    scheduler = get_linear_schedule_with_warmup(optimizer,
+                                                num_warmup_steps=0, # Default value
+                                                num_training_steps=total_steps)
+    return bert_regressor, optimizer, scheduler
+import random
+import time
+
+# Specify loss function
+loss_fn = nn.MSELoss()
+
+def set_seed(seed_value=42):
+    """Set seed for reproducibility.
+    """
+    random.seed(seed_value)
+    np.random.seed(seed_value)
+    torch.manual_seed(seed_value)
+    torch.cuda.manual_seed_all(seed_value)
+
+def train(model, I_trn,B_trn,M_trn,I_tst,B_tst,M_tst,
+          batch_size_tst=32, batch_size=50,batch_epochs=400, evaluation=False,batch_size_trn=32):
+        """Train the BertClassifier model.
+        """
+        #initialize val_loss with something big to prevent initialization error
+#         val_loss=10
+        # Start training loop
+        print("Start training...\n")
+        # =======================================
+        #               Training
+        # =======================================
+        # Print the header of the result table
+        print(f" {'Batch':^5} | {'Train Loss':^12} | {'Val Loss':^10}  | {'Elapsed':^9}")
+        print("-"*50)
+        # Measure the elapsed time of each epoch
+        t0_batch = time.time()
+        # Reset tracking variables at the beginning of each epoch
+        batch_loss, batch_counts =  0, 0
+        # Put the model into the training mode
+        model.train()
+        # For each batch of training data...
+        for batch in range(batch_epochs):  #298
+            batch_counts +=1
+            if ((batch==(704))):break  #457#383#1451#246
+#             if val_loss<0.3:      break
+            # Load batch to GPU
+            b_input_ids, b_attn_mask, b_ys,_ = tuple(t.to(device) for t in dta_ldr(I=I_trn,B=B_trn,M=M_trn,batch_size=batch_size_trn))
+            # Zero out any previously calculated gradients
+            model.zero_grad()
+            # Perform a forward pass. This will return logits.
+#             print(b_input_ids,'Mask:\n',b_attn_mask)
+            preds = model(b_input_ids, b_attn_mask)
+            # Compute loss 
+            loss = loss_fn(preds.float(), b_ys.float())
+            batch_loss += loss.item()
+            # Perform a backward pass to calculate gradients
+            loss.backward()
+            # Clip the norm of the gradients to 1.0 to prevent "exploding gradients"
+            torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)
+            # Update parameters and the learning rate
+            optimizer.step()
+            scheduler.step()
+
+            # Print the loss values and time elapsed for every 20 batches
+            if (batch_counts % 50 == 0 and batch_counts != 0) : #or(batch>585)
+                # Calculate time elapsed for 20 batches
+                time_elapsed = time.time() - t0_batch
+
+                # Print training results
+                val_loss = evaluate(model, Ie=I_tst,Be=B_tst,Me=M_tst,batch_size_e=batch_size_tst)
+                print(f"{batch+ 1:^7}|{batch_loss / batch_counts:^12.6f} |  {val_loss:^10.6f}  | {time_elapsed:^9.2f}") #| {step:^7}
+                # After the completion of each training epoch, measure the model's performance
+                # on our validation set.
+                print("-"*50)
+#                 print(batch)
+
+#                 if (batch<586):
+#                 # Reset batch tracking variables
+#                     batch_loss, batch_counts = 0, 0
+#                     t0_batch = time.time()
+#                 # Reset batch tracking variables
+                batch_loss, batch_counts = 0, 0
+                t0_batch = time.time()
+
+            # Calculate the average loss over the entire training data
+#             avg_train_loss = total_loss / (batch_size*batch_epochs)
+
+            # =======================================
+            #               Evaluation
+            # =======================================
+            if evaluation == True:
+                # After the completion of each training epoch, measure the model's performance
+                # on our validation set.
+                val_loss = evaluate(model, Ie=I_tst,Be=B_tst,Me=M_tst,batch_size_e=batch_size_tst)
+                if val_loss<0.32: 
+                    print('\n Consider this one with val:', val_loss,' at:',batch,'\n')
+                    print("-"*50)
+
+
+            
+                    # Calculate the average loss over the entire training data
+#         avg_train_loss = total_loss / (batch_size*batch_epochs)
+
+        val_loss = evaluate(model, Ie=I_tst,Be=B_tst,Me=M_tst,batch_size_e=batch_size_tst)
+        print(f"{batch+ 1:^7}|{batch_loss / batch_counts:^12.6f} |  {val_loss:^10.6f}  | {time_elapsed:^9.2f}") #| {step:^7}   
+        print("Training complete!")
+
+
+def evaluate(model, Ie,Be,Me,batch_size_e):
+    """After the completion of each training epoch, measure the model's performance
+    on our validation set.
+    """
+    # Put the model into the evaluation mode. The dropout layers are disabled during
+    # the test time.
+    model.eval()
+
+    # Tracking variables
+    val_loss = []
+
+    # For each batch in our validation set...
+    for batch in range(1):
+        # Load batch to GPU
+        b_input_ids, b_attn_mask, b_ys,_  = tuple(t.to(device) for t in dta_ldr2(Ie,Be,Me,batch_size_e))
+
+        # Compute logits
+        with torch.no_grad():
+            preds = model(b_input_ids, b_attn_mask)
+
+        # Compute loss
+        loss = loss_fn(preds, b_ys)
+        val_loss.append(loss.item())
+
+
+    # Compute the absolutr error and loss over the validation set.
+    val_loss = np.mean(val_loss)
+
+    return val_loss
+
+
+rnd_st=42
+np.random.seed(rnd_st)
+random.seed(rnd_st)
+torch.manual_seed(rnd_st)
+torch.cuda.manual_seed(rnd_st)
+# Running on the CuDNN backend
+torch.backends.cudnn.deterministic = True
+torch.backends.cudnn.benchmark = False
+
+
+## ****** Load pre-trained model  ******** ##
+bert_regressor = BertRegressor()
+bert_regressor.load_state_dict(torch.load("BERTNN_model",map_location=torch.device(device)))
+bert_regressor.eval()
+
+
+
+
+def bert_predict(model, test_dataloader):
+    """Perform a forward pass on the trained BERT model to predict probabilities
+    on the test set.
+    """
+    # Put the model into the evaluation mode. The dropout layers are disabled during
+    # the test time.
+    model.eval()
+    all_preds = []
+    # For each batch in our test set...
+    for batch in range(1):
+        # Load batch to GPU
+        b_input_ids, b_attn_mask = tuple(t.to(device) for t in test_dataloader)[:2]
+
+        # Compute predictions
+        with torch.no_grad():
+            preds = model(b_input_ids, b_attn_mask)#.to(device)
+        all_preds.append(preds)
+    
+    # Concatenate predictions from each batch
+    all_preds = torch.cat(all_preds, dim=0)
+
+    return all_preds
+
+## ****** The following function inverses the normalization function and uses original dictionaries to represent MABMO events  ******** ##
+
+def out_df(data,predictions,df_beh=Behaviors,df_ident=Identities,df_mod=Modifiers):
+    df2=pd.concat([pd.DataFrame(scaler_M.inverse_transform(predictions[:,0:3].cpu())),
+                   pd.DataFrame(scaler_M.inverse_transform(data[2][:,0:3])),
+                   pd.DataFrame(scaler_I.inverse_transform(predictions[:,3:6].cpu())),
+                   pd.DataFrame(scaler_I.inverse_transform(data[2][:,3:6])),
+                    pd.DataFrame(scaler_B.inverse_transform(predictions[:,6:9].cpu())),
+                   pd.DataFrame(scaler_B.inverse_transform(data[2][:,6:9])),
+                    pd.DataFrame(scaler_M.inverse_transform(predictions[:,9:12].cpu())),
+                   pd.DataFrame(scaler_M.inverse_transform(data[2][:,9:12])),
+                    pd.DataFrame(scaler_I.inverse_transform(predictions[:,12:15].cpu())),
+                   pd.DataFrame(scaler_I.inverse_transform(data[2][:,12:15])),pd.DataFrame(np.array(data[3]))
+                  ],axis=1).set_axis(['EEMA', 'EPMA', 'EAMA','EM1', 'PM1', 'AM1',
+                                      'EEA', 'EPA', 'EAA','EA', 'PA', 'AA',
+                                      'EEB', 'EPB', 'EAB','EB', 'PB', 'AB',
+                                      'EEMO', 'EPMO', 'EAMO','EM2', 'PM2', 'AM2',
+                                      'EEO', 'EPO', 'EAO','EO', 'PO', 'AO',
+                                      'idx_ModA','idx_Act','idx_Beh','idx_ModO','idx_Obj'], axis=1, inplace=False)
+    df2=pd.merge(df2, df_mod[['term','index_in_dic']], left_on= ['idx_ModA'], right_on = ["index_in_dic"], 
+          how='left').rename(columns={"term": 'ModA'}).drop(['index_in_dic'], axis=1)
+    df2=pd.merge(df2, df_ident[['term','index_in_dic']], left_on= ['idx_Act'], right_on = ["index_in_dic"], 
+          how='left').rename(columns={"term": 'Actor'}).drop(['index_in_dic'], axis=1)
+    df2=pd.merge(df2, df_beh[['term','index_in_dic']], left_on= ['idx_Beh'], right_on = ["index_in_dic"], 
+          how='left').rename(columns={"term": 'Behavior'}).drop(['index_in_dic'], axis=1)
+    df2=pd.merge(df2, df_mod[['term','index_in_dic']], left_on= ['idx_ModO'], right_on = ["index_in_dic"], 
+          how='left').rename(columns={"term": 'ModO'}).drop(['index_in_dic'], axis=1)
+    df2=pd.merge(df2, df_ident[['term','index_in_dic']], left_on= ['idx_Obj'], right_on = ["index_in_dic"], 
+          how='left').rename(columns={"term": 'Object'}).drop(['index_in_dic'], axis=1)
+
+    df2=df2[['EEMA','EPMA', 'EAMA', 'EEA', 'EPA', 'EAA', 'EEB', 'EPB', 'EAB','EEMO', 'EPMO', 'EAMO', 'EEO', 'EPO', 'EAO','EM1', 'PM1', 'AM1','EA', 'PA', 'AA',  'EB', 'PB','AB',  'EM2', 'PM2', 'AM2', 'EO',
+           'PO', 'AO', 'ModA','Actor','Behavior', 'ModO', 'Object']]
+    return(df2)
+
+def get_output(I_b=n_Identities,B_b=n_Behaviors,M_b=n_Modifiers,batch_sz=3000,batch_num=10):
+    df=pd.DataFrame()
+    for i in range(batch_num):
+        q=dta_ldr2(I=I_b,B=B_b,M=M_b,batch_size=batch_sz)
+        preds = bert_predict(bert_regressor.to(device), q)
+        df2=out_df(data=q,predictions=preds)
+        df=pd.concat([df,df2],axis=0)
+    return(df)
+
+
+def gen_new(Identity,Behavior,Modifier,n_df,word_type):
+        if word_type=='identity':
+            ident1=n_df.sample(axis = 0,random_state=56)
+        else:ident1=Identity.sample(axis = 0,random_state=6)
+        ident2=Identity.sample(axis = 0,random_state=6)
+        if word_type=='behavior':
+            behav=n_df.sample(axis = 0,random_state=5)
+        else: behav=Behavior.sample(axis = 0,random_state=5)
+        if word_type=='modifier':
+            modif1=n_df.sample(axis = 0,random_state=55)
+        else:    modif1=Modifier.sample(axis = 0)
+        modif2=Modifier.sample(axis = 0,random_state=96)
+        id1=list(ident1.term)
+        id2=list(ident2.term)
+        beh=list(behav.term)
+        mod1=list(modif1.term)
+        mod2=list(modif2.term)
+#         wrdvc_ident1=gs_model.get_vector((list(ident1.trm_org))[0], norm=True)
+        sents=' '.join(map(str, (mod1+id1+beh+mod2+id2)))
+        values=np.concatenate([(modif1[['E','P','A']]).to_numpy(),
+                               (ident1[['E','P','A']]).to_numpy(),
+                              (behav[['E','P','A']]).to_numpy(),
+                               (modif2[['E','P','A']]).to_numpy(),
+                               (ident2[['E','P','A']]).to_numpy()], axis=1)[0]
+        #indexx=[(ident1['index_in_dic']).to_numpy()][0][0]
+        indexx=torch.tensor([[(modif1['index_in_dic']).to_numpy()][0][0],
+                            [(ident1['index_in_dic']).to_numpy()][0][0],
+                            [(behav['index_in_dic']).to_numpy()][0][0],
+                            [(modif2['index_in_dic']).to_numpy()][0][0],
+                            [(ident2['index_in_dic']).to_numpy()][0][0]])
+        ys= torch.tensor(values)
+        inputs, masks = preprocessing_for_bert([sents])
+        yield inputs, masks, ys,indexx  #torch.tensor(sents),
+def ldr_new(I,B,M,N_df,WT,batch_size=32):
+    dt_ldr= [x for x in DataLoader([next(gen_new(I,B,M,N_df,WT)) for x in range(batch_size)],  batch_size=batch_size)][0]
+    return(dt_ldr)
+
+def gen_new(Identity,Behavior,Modifier,n_df,word_type):
+
+        modif1,modif2,ident1,ident2,behav=Modifier.sample(axis = 0),Modifier.sample(axis = 0),Identity.sample(axis = 0),Identity.sample(axis = 0),Behavior.sample(axis = 0)
+
+        if word_type=='identity':   ident1=n_df.sample(axis = 0)
+        if word_type=='behavior':   behav=n_df.sample(axis = 0)
+        if word_type=='modifier':  modif1=n_df.sample(axis = 0)
+        
+        id1,id2,beh,mod1,mod2=list(ident1.term),list(ident2.term),list(behav.term),list(modif1.term),list(modif2.term)
+
+#         wrdvc_ident1=gs_model.get_vector((list(ident1.trm_org))[0], norm=True)
+        sents=' '.join(map(str, (mod1+id1+beh+mod2+id2)))
+        values=np.concatenate([(modif1[['E','P','A']]).to_numpy(),
+                               (ident1[['E','P','A']]).to_numpy(),
+                              (behav[['E','P','A']]).to_numpy(),
+                               (modif2[['E','P','A']]).to_numpy(),
+                               (ident2[['E','P','A']]).to_numpy()], axis=1)[0]
+        indexx=torch.tensor([[(modif1['index_in_dic']).to_numpy()][0][0],
+                            [(ident1['index_in_dic']).to_numpy()][0][0],
+                            [(behav['index_in_dic']).to_numpy()][0][0],
+                            [(modif2['index_in_dic']).to_numpy()][0][0],
+                            [(ident2['index_in_dic']).to_numpy()][0][0]])
+        ys= torch.tensor(values)
+        inputs, masks = preprocessing_for_bert([sents])
+        yield inputs, masks, ys,indexx  
+
+
+def gen_alt(Identity,Behavior,Modifier,n_df,word_type):
+
+        modif1,modif2,ident1,ident2,behav=Modifier.sample(axis = 0),Modifier.sample(axis = 0),Identity.sample(axis = 0),Identity.sample(axis = 0),Behavior.sample(axis = 0)
+        if word_type=='identity':   ident2=n_df.sample(axis = 0)
+        if word_type=='behavior':   behav=n_df.sample(axis = 0)
+        if word_type=='modifier':  modif2=n_df.sample(axis = 0)
+        
+        id1,id2,beh,mod1,mod2=list(ident1.term),list(ident2.term),list(behav.term),list(modif1.term),list(modif2.term)
+        sents=' '.join(map(str, (mod1+id1+beh+mod2+id2)))
+        values=np.concatenate([(modif1[['E','P','A']]).to_numpy(),
+                               (ident1[['E','P','A']]).to_numpy(),
+                              (behav[['E','P','A']]).to_numpy(),
+                               (modif2[['E','P','A']]).to_numpy(),
+                               (ident2[['E','P','A']]).to_numpy()], axis=1)[0]
+        indexx=torch.tensor([[(modif1['index_in_dic']).to_numpy()][0][0],
+                            [(ident1['index_in_dic']).to_numpy()][0][0],
+                            [(behav['index_in_dic']).to_numpy()][0][0],
+                            [(modif2['index_in_dic']).to_numpy()][0][0],
+                            [(ident2['index_in_dic']).to_numpy()][0][0]])
+        ys= torch.tensor(values)
+        inputs, masks = preprocessing_for_bert([sents])
+
+        yield inputs, masks, ys,indexx   
+        
+def ldr_new(I,B,M,N_df,WT,batch_size=32,alt=0):
+    if alt:
+        dt_ldr= [x for x in DataLoader([next(gen_alt(I,B,M,N_df,WT)) for x in range(batch_size)],  batch_size=batch_size)][0]
+    else:
+        dt_ldr= [x for x in DataLoader([next(gen_new(I,B,M,N_df,WT)) for x in range(batch_size)],  batch_size=batch_size)][0]
+    return(dt_ldr)
+
+cols=['EEMA', 'EPMA', 'EAMA', 'EEA', 'EPA', 'EAA', 'EEB', 'EPB', 'EAB',
+       'EEMO', 'EPMO', 'EAMO', 'EEO', 'EPO', 'EAO',  'ModA', 'Actor', 'Behavior', 'ModO', 'Object']
+def get_output_new(w,wt,I_b=n_Identities,B_b=n_Behaviors,M_b=n_Modifiers,batch_sz=300,batch_num=1,columnss=cols,cus_col=1):
+    
+    df=pd.DataFrame()
+    for i in range(batch_num):
+        new_df=pd.DataFrame({'index_in_dic':4000,'term':w,'E':10,'P':10,'A':10,'E2':10,'P2':10,'A2':10,'term2':w,'len_Bert':3}, index=[0])
+        q=ldr_new(I=I_b,B=B_b,M=M_b,N_df=new_df,WT=wt,batch_size=batch_sz)
+        preds = bert_predict(bert_regressor.to(device), q)
+        if wt=='identity':
+            df_identity=pd.concat([Identities,new_df],axis=0)
+            df2=out_df(data=q,predictions=preds,df_ident=df_identity)
+            if cus_col:
+                columnss=[ 'EEA', 'EPA', 'EAA', 'ModA', 'Actor', 'Behavior', 'ModO', 'Object']
+        if wt=='behavior': 
+            df_behavior=pd.concat([Behaviors,new_df],axis=0)
+            df2=out_df(data=q,predictions=preds,df_beh=df_behavior)
+            if cus_col:
+                columnss=['EEB', 'EPB', 'EAB',  'ModA', 'Actor', 'Behavior', 'ModO', 'Object']            
+        if wt=='modifier': 
+            df_modifier=pd.concat([Modifiers,new_df],axis=0)
+            df2=out_df(data=q,predictions=preds,df_mod=df_modifier)
+            if cus_col:
+                columnss=['EEMA', 'EPMA', 'EAMA',  'ModA', 'Actor', 'Behavior', 'ModO', 'Object']
+        df=pd.concat([df,df2],axis=0)
+    return(df[columnss])
+
+def gen_new(Identity,Behavior,Modifier,n_df,word_type):
+        if word_type=='identity':
+            ident1=n_df.sample(axis = 0)
+        else:ident1=Identity.sample(axis = 0)
+        ident2=Identity.sample(axis = 0)
+        if word_type=='behavior':
+            behav=n_df.sample(axis = 0)
+        else: behav=Behavior.sample(axis = 0)
+        if word_type=='modifier':
+            modif1=n_df.sample(axis = 0)
+        else:    modif1=Modifier.sample(axis = 0)
+        modif2=Modifier.sample(axis = 0)
+        id1=list(ident1.term)
+        id2=list(ident2.term)
+        beh=list(behav.term)
+        mod1=list(modif1.term)
+        mod2=list(modif2.term)
+        sents=' '.join(map(str, (mod1+id1+beh+mod2+id2)))
+        values=np.concatenate([(modif1[['E','P','A']]).to_numpy(),
+                               (ident1[['E','P','A']]).to_numpy(),
+                              (behav[['E','P','A']]).to_numpy(),
+                               (modif2[['E','P','A']]).to_numpy(),
+                               (ident2[['E','P','A']]).to_numpy()], axis=1)[0]
+        indexx=torch.tensor([[(modif1['index_in_dic']).to_numpy()][0][0],
+                            [(ident1['index_in_dic']).to_numpy()][0][0],
+                            [(behav['index_in_dic']).to_numpy()][0][0],
+                            [(modif2['index_in_dic']).to_numpy()][0][0],
+                            [(ident2['index_in_dic']).to_numpy()][0][0]])
+        ys= torch.tensor(values)
+        inputs, masks = preprocessing_for_bert([sents])
+        yield inputs, masks, ys,indexx  #torch.tensor(sents),
+
+def get_output_agg(w,wt,I_b=n_I_v,B_b=n_B_v,M_b=n_M_v,batch_sz=300,batch_num=1):   
+    df=pd.DataFrame()
+    for i in range(batch_num):
+        new_df=pd.DataFrame({'index_in_dic':4000,'term':w,'E':10,'P':10,'A':10,'E2':10,'P2':10,'A2':10,'term2':w,'len_Bert':3}, index=[0])
+        batch1=int(batch_sz/2)
+        batch2=batch_sz-batch1
+        if wt=='behavior': 
+            q=ldr_new(I=I_b,B=B_b,M=M_b,N_df=new_df,WT=wt,batch_size=batch_sz)
+            preds = bert_predict(bert_regressor.to(device), q)
+            df_behavior=pd.concat([Behaviors,new_df],axis=0)
+            df2=out_df(data=q,predictions=preds,df_beh=df_behavior)
+            df=pd.concat([df,df2],axis=0)[['EEB','EPB','EAB','EB','PB', 'AB','ModA', 'Actor', 'Behavior', 'ModO', 'Object']].rename(columns={'EEB':'EE','EPB':'EP','EAB':'EA','EB':'E','PB':'P', 'AB':'A'})    
+
+        if wt=='identity':
+            q=ldr_new(I=I_b,B=B_b,M=M_b,N_df=new_df,WT=wt,batch_size=batch1)
+            preds = bert_predict(bert_regressor.to(device), q)
+            df_identity=pd.concat([Identities,new_df],axis=0)
+            df2=out_df(data=q,predictions=preds,df_ident=df_identity)
+            df_act=pd.concat([df,df2],axis=0)
+            df_act=df_act.copy()[['EEA','EPA','EAA','EA','PA', 'AA','ModA', 'Actor', 'Behavior', 'ModO', 'Object']].rename(columns={'EEA':'EE','EPA':'EP','EAA':'EA','EA':'E','PA':'P', 'AA':'A'})
+            q=ldr_new(I=I_b,B=B_b,M=M_b,N_df=new_df,WT=wt,batch_size=batch2,alt=1)
+            preds = bert_predict(bert_regressor.to(device), q)
+            df_identity=pd.concat([Identities,new_df],axis=0)
+            df2=out_df(data=q,predictions=preds,df_ident=df_identity)
+            df_obj=pd.concat([df,df2],axis=0)
+            df_obj=df_obj.copy()[['EEO','EPO','EAO','EO', 'PO', 'AO','ModA', 'Actor', 'Behavior', 'ModO', 'Object']].rename(columns={'EEO':'EE','EPO':'EP','EAO':'EA','EO':'E','PO':'P', 'AO':'A'})
+            df=pd.concat([df_act,df_obj],axis=0)
+        if wt=='modifier': 
+            q=ldr_new(I=I_b,B=B_b,M=M_b,N_df=new_df,WT=wt,batch_size=batch1)
+            preds = bert_predict(bert_regressor.to(device), q)
+            df_modifier=pd.concat([Modifiers,new_df],axis=0)
+            df2=out_df(data=q,predictions=preds,df_mod=df_modifier)
+            df_act=pd.concat([df,df2],axis=0)    
+            df_act=df_act.copy()[['EEMA', 'EPMA', 'EAMA','EM1', 'PM1', 'AM1','ModA', 'Actor', 'Behavior', 'ModO', 'Object']].rename(columns={'EEMA':'EE','EPMA':'EP','EAMA':'EA','EM1':'E','PM1':'P', 'AM1':'A'})
+            q=ldr_new(I=I_b,B=B_b,M=M_b,N_df=new_df,WT=wt,batch_size=batch2,alt=1)
+            preds = bert_predict(bert_regressor.to(device), q)
+            df_modifier=pd.concat([Modifiers,new_df],axis=0)
+            df2=out_df(data=q,predictions=preds,df_mod=df_modifier)
+            df_obj=pd.concat([df,df2],axis=0)
+            df_obj=df_obj.copy()[['EEMO', 'EPMO', 'EAMO','EM2', 'PM2', 'AM2','ModA', 'Actor', 'Behavior', 'ModO', 'Object']].rename(columns={'EEMO':'EE','EPMO':'EP','EAMO':'EA','EM2':'E','PM2':'P', 'AM2':'A'})
+            df=pd.concat([df_act,df_obj],axis=0)        
+    return(df)
+
+def ldr_new(I,B,M,N_df,WT,batch_size=32):
+    dt_ldr= [x for x in DataLoader([next(gen_new(I,B,M,N_df,WT)) for x in range(batch_size)],  batch_size=batch_size)][0]
+    return(dt_ldr)
+
+
+
+def sent_gen(sentence):
+        sents=sentence
+        indexx=torch.tensor([1,1,1,1,1,1,1,1,1,1,1,1])
+        ys= torch.tensor([1,1,1,1,1,1,1,1,1,1,1,1])
+        inputs, masks = preprocessing_for_bert([sents])
+        yield inputs, masks, ys,indexx  #torch.tensor(sents),
+def sent_ldr(sent2,batch_size=1):
+    dt_ldr= [x for x in DataLoader([next(sent_gen(sent2)) for x in range(batch_size)],  batch_size=batch_size)][0]
+    return(dt_ldr)
+def EPA_sents(sent):
+    q=sent_ldr(sent)
+    predictions=bert_predict(bert_regressor.to(device), q)
+    df_out=pd.concat([pd.DataFrame(scaler_M.inverse_transform(predictions[:,0:3].cpu())),
+               pd.DataFrame(scaler_I.inverse_transform(predictions[:,3:6].cpu())),
+               pd.DataFrame(scaler_B.inverse_transform(predictions[:,6:9].cpu())),
+               pd.DataFrame(scaler_M.inverse_transform(predictions[:,9:12].cpu())),
+               pd.DataFrame(scaler_I.inverse_transform(predictions[:,12:15].cpu()))
+                                    ],axis=1).set_axis(['EEMA', 'EPMA', 'EAMA',
+                                          'EEA', 'EPA', 'EAA',  'EEB', 'EPB', 'EAB',
+                          'EEMO', 'EPMO', 'EAMO','EEO', 'EPO', 'EAO'], axis=1, inplace=False)
+    return(df_out.round(decimals=2))
+
+# Ref: https://stackoverflow.com/questions/28778668/freeze-header-in-pandas-dataframe
+
+from ipywidgets import interact, IntSlider
+from IPython.display import display
+
+def freeze_header(df, num_rows=30, num_columns=10, step_rows=1,
+                  step_columns=1):
+    """
+    Freeze the headers (column and index names) of a Pandas DataFrame. A widget
+    enables to slide through the rows and columns.
+    Parameters
+    ----------
+    df : Pandas DataFrame
+        DataFrame to display
+    num_rows : int, optional
+        Number of rows to display
+    num_columns : int, optional
+        Number of columns to display
+    step_rows : int, optional
+        Step in the rows
+    step_columns : int, optional
+        Step in the columns
+    Returns
+    -------
+    Displays the DataFrame with the widget
+    """
+    @interact(last_row=IntSlider(min=min(num_rows, df.shape[0]),
+                                 max=df.shape[0],
+                                 step=step_rows,
+                                 description='rows',
+                                 readout=False,
+                                 disabled=False,
+                                 continuous_update=True,
+                                 orientation='horizontal',
+                                 slider_color='purple'),
+              last_column=IntSlider(min=min(num_columns, df.shape[1]),
+                                    max=df.shape[1],
+                                    step=step_columns,
+                                    description='columns',
+                                    readout=False,
+                                    disabled=False,
+                                    continuous_update=True,
+                                    orientation='horizontal',
+                                    slider_color='purple'))
+    def _freeze_header(last_row, last_column):
+        display(df.iloc[max(0, last_row-num_rows):last_row,
+                        max(0, last_column-num_columns):last_column])
+        
+
+
+