Upload folder using huggingface_hub

modeling_roberta_cl.py

@@ -2,35 +2,10 @@ import torch
 import torch.nn as nn
 import torch.nn.functional as F
 import torch.distributed as dist
-from torch import Tensor
 
 import transformers
-from transformers import RobertaTokenizer
-from transformers.models.roberta.modeling_roberta import RobertaForSequenceClassification, RobertaClassificationHead, RobertaLMHead
-from transformers.activations import gelu
-from transformers.file_utils import (
-    add_code_sample_docstrings,
-    add_start_docstrings,
-    add_start_docstrings_to_model_forward,
-    replace_return_docstrings,
-)
-from transformers.modeling_outputs import SequenceClassifierOutput, BaseModelOutputWithPoolingAndCrossAttentions
-
-class MLPLayer(nn.Module):
-    """
-    Head for getting sentence representations over RoBERTa/BERT's CLS representation.
-    """
-
-    def __init__(self, config):
-        super().__init__()
-        self.dense = nn.Linear(config.hidden_size, config.hidden_size)
-        self.activation = nn.Tanh()
-
-    def forward(self, features, **kwargs):
-        x = self.dense(features)
-        x = self.activation(x)
-
-        return x
+from transformers.models.roberta.modeling_roberta import RobertaForSequenceClassification, RobertaClassificationHead
+from transformers.modeling_outputs import BaseModelOutputWithPoolingAndCrossAttentions
 
 class ResidualBlock(nn.Module):
     def __init__(self, dim):
@@ -129,8 +104,6 @@ def cl_forward(cls,
     output_attentions=None,
     output_hidden_states=None,
     return_dict=None,
-    mlm_input_ids=None,
-    mlm_labels=None,
     latter_sentiment_spoof_mask=None,
 ):
     return_dict = return_dict if return_dict is not None else cls.config.use_return_dict
@@ -139,7 +112,6 @@ def cl_forward(cls,
     # original + cls.model_args.num_paraphrased + cls.model_args.num_negative
     num_sent = input_ids.size(1)
 
-    mlm_outputs = None
     # Flatten input for encoding
     input_ids = input_ids.view((-1, input_ids.size(-1))) # (bs * num_sent, len)
     attention_mask = attention_mask.view((-1, attention_mask.size(-1))) # (bs * num_sent len)
@@ -159,21 +131,6 @@ def cl_forward(cls,
         return_dict=True,
     )
 
-    # MLM auxiliary objective
-    if mlm_input_ids is not None:
-        mlm_input_ids = mlm_input_ids.view((-1, mlm_input_ids.size(-1)))
-        mlm_outputs = cls.roberta(
-            mlm_input_ids,
-            attention_mask=attention_mask,
-            token_type_ids=token_type_ids,
-            position_ids=position_ids,
-            head_mask=head_mask,
-            inputs_embeds=inputs_embeds,
-            output_attentions=output_attentions,
-            output_hidden_states=False,
-            return_dict=True,
-        )
-
     # Pooling
     sequence_output = outputs[0]  # (bs*num_sent, seq_len, hidden)
     pooler_output = cls.classifier(sequence_output)  # (bs*num_sent, hidden)
@@ -216,14 +173,6 @@ def cl_forward(cls,
                 loss_triplet += F.relu(cls.sim(ori, neg) * cls.model_args.temp  - cls.sim(ori, pos) * cls.model_args.temp  + cls.model_args.margin)
     loss_triplet /= (batch_size * cls.model_args.num_paraphrased * len(spoofing_cnames))
 
-    # Calculate loss for MLM
-    if mlm_outputs is not None and mlm_labels is not None:
-        raise NotImplementedError
-        # mlm_labels = mlm_labels.view(-1, mlm_labels.size(-1))
-        # prediction_scores = cls.lm_head(mlm_outputs.last_hidden_state)
-        # masked_lm_loss = loss_fct(prediction_scores.view(-1, cls.config.vocab_size), mlm_labels.view(-1))
-        # loss_cl = loss_cl + cls.model_args.mlm_weight * masked_lm_loss
-    
     # Calculate loss for uniform perturbation and unbiased token preference
     def sign_loss(x):
         row = torch.abs(torch.mean(torch.mean(x, dim=0)))
@@ -331,8 +280,7 @@ class RobertaForCL(RobertaForSequenceClassification):
 
         self.classifier = RobertaClassificationHeadForEmbedding(config)
 
-        if self.model_args and getattr(self.model_args, "do_mlm", False):
-            self.lm_head = RobertaLMHead(config)
+        if self.model_args:
             cl_init(self, config)
 
         self.map = SemanticModel(input_dim=768)
@@ -340,13 +288,6 @@ class RobertaForCL(RobertaForSequenceClassification):
         # Initialize weights and apply final processing
         self.post_init()
 
-    def initialize_mlp_weights(self, pretrained_model_state_dict):
-        """
-        Initialize MLP weights using the pretrained classifier's weights.
-        """
-        self.mlp.dense.weight.data = pretrained_model_state_dict.classifier.dense.weight.data.clone()
-        self.mlp.dense.bias.data = pretrained_model_state_dict.classifier.dense.bias.data.clone()
-
     def forward(self,
         input_ids=None,
         attention_mask=None,
@@ -359,8 +300,6 @@ class RobertaForCL(RobertaForSequenceClassification):
         output_hidden_states=None,
         return_dict=None,
         sent_emb=False,
-        mlm_input_ids=None,
-        mlm_labels=None,
         latter_sentiment_spoof_mask=None,
     ):
         if sent_emb:
@@ -388,8 +327,6 @@ class RobertaForCL(RobertaForSequenceClassification):
                 output_attentions=output_attentions,
                 output_hidden_states=output_hidden_states,
                 return_dict=return_dict,
-                mlm_input_ids=mlm_input_ids,
-                mlm_labels=mlm_labels,
                 latter_sentiment_spoof_mask=latter_sentiment_spoof_mask,
             )