Delete ISPIn/ISPIn.py

ISPIn/ISPIn.py

@@ -1,128 +0,0 @@
-# SYS
-import gc
-
-# BASE
-import numpy as np
-
-# TORCH
-import torch
-from torch import nn
-
-# TQDM
-from tqdm.notebook import tqdm
-from tqdm.autonotebook import trange
-
-# TRANSFORMERS
-from transformers import PreTrainedModel, AutoTokenizer, AutoModel
-
-# CONFIG
-from .ISPInConfig import ISPInConfig
-
-class ISPIn(PreTrainedModel):
-    
-    config_class = ISPInConfig
-    
-    def __init__(self, config):
-        super(ISPIn, self).__init__(config=config)
-        
-        self.tokenizer = AutoTokenizer.from_pretrained(config.sbert_id)
-        self.encoder = AutoModel.from_pretrained(config.sbert_id)
-
-        self.layers = nn.ModuleList()
-        
-        for index in range(len(config.layers_dimensions) - 1):
-            self.layers.append(nn.Linear(config.layers_dimensions[index], config.layers_dimensions[index + 1]))
-
-        self.activation = nn.GELU()
-
-        self.local_labels_dict = {'contradiction': 2, 'neutral': 0, 'entailment': 1}
-       
-    
-    def mean_pooling(self, x, attention_mask):
-        token_embeddings = x[0] #First element of SBERT output contains all token embeddings
-        input_mask_expanded = attention_mask.unsqueeze(-1).expand(token_embeddings.size()).float()
-        return torch.sum(token_embeddings * input_mask_expanded, 1) / torch.clamp(input_mask_expanded.sum(1), min=1e-9)
-
-
-    def forward(self, sentences, batch_size = 32, show_progress_bar = False):
-
-        # cast individual sentence to a list of sentences of length one
-        if isinstance(sentences, str) or not hasattr(sentences, '__len__'):
-            sentences = [sentences]
-
-        all_embeddings = []
-
-        for start_index in trange(0, len(sentences), batch_size, desc="Batches", disable=not show_progress_bar):
-            sentences_batch = sentences[start_index : start_index + batch_size]
-        
-            tokenized_batch = self.tokenizer(sentences_batch, padding=True, truncation=True, return_tensors='pt')
-
-            with torch.no_grad():
-                last_layer = self.encoder(**tokenized_batch)
-                embeddings = self.mean_pooling(last_layer, tokenized_batch['attention_mask'])
-
-            all_embeddings.extend(embeddings)
-            del embeddings
-            gc.collect()
-
-        all_embeddings = torch.stack(all_embeddings)
-
-        return all_embeddings
-
-
-    def predict(self, embeddings_to_predict, embeddings_ground_truth, one_to_many = False):
-        
-        if (one_to_many == False):
-
-            prod = torch.mul(embeddings_to_predict, embeddings_ground_truth)
-            diff = torch.subtract(embeddings_to_predict, embeddings_ground_truth)
-
-            prod_diff = torch.concat((prod, diff), 1)
-
-            x = prod_diff
-            del prod_diff
-
-        elif (one_to_many == True):
-            
-            number_of_sentences = embeddings_to_predict.size(dim = 0)
-            number_of_ground_truths = embeddings_ground_truth.size(dim = 0)
-            embedding_dimension = embeddings_to_predict.size(dim = 1)
-
-            prod_diff = torch.zeros((number_of_sentences * number_of_ground_truths, 2 * embedding_dimension))
-        
-            for i in range(number_of_sentences):
-                start_index = i * number_of_ground_truths
-                end_index = (i + 1) * number_of_ground_truths
-
-                # shape (1, embedding_dimension)
-                single_sentence_embeddings = torch.reshape(embeddings_to_predict[i], (1, embedding_dimension))
-
-                prod = torch.mul(single_sentence_embeddings, embeddings_ground_truth)
-                diff = torch.subtract(single_sentence_embeddings, embeddings_ground_truth)
-
-                prod_diff[start_index : end_index] = torch.concat((prod, diff), axis=1)
-
-            x = prod_diff
-            del prod_diff
-
-        del prod, diff
-
-        for layer in self.layers:
-            x = self.activation(layer(x))
-        
-        if (one_to_many == False):
-            probabilities = [torch.softmax(elem, 0) for elem in x]
-            predictions = [torch.argmax(elem, 0) for elem in x]
-        
-        if (one_to_many == True):
-
-            # Reshape as a matrix. predictions[0] are the labels given to sentence[0] with all ground truths 
-            x = [x[i * number_of_ground_truths : (i + 1) * number_of_ground_truths] for i in range(number_of_sentences)]
-
-            probabilities = [torch.softmax(elem, 1) for elem in x]
-            predictions = [torch.argmax(elem, 1) for elem in x]
-
-        predictions = np.array([prediction.cpu().detach().numpy() for prediction in predictions])
-        probabilities = np.array([probability.cpu().detach().numpy() for probability in probabilities])
-
-        return predictions