Upload modeling.py with huggingface_hub

modeling.py

@@ -0,0 +1,97 @@
+
+from transformers.models.bert.modeling_bert import BertPreTrainedModel, BertModel
+from torch import nn
+
+class IntentClassifier(nn.Module):
+    def __init__(self, input_dim, num_intent_labels, dropout_rate=0.):
+        super(IntentClassifier, self).__init__()
+        self.dropout = nn.Dropout(dropout_rate)
+        self.linear = nn.Linear(input_dim, num_intent_labels)
+        
+    def forward(self, x):
+        x = self.dropout(x)
+        return self.linear(x)
+
+class SlotClassifier(nn.Module):
+    def __init__(self, input_dim, num_slot_labels, dropout_rate=0.):
+        super(SlotClassifier, self).__init__()
+        self.dropout = nn.Dropout(dropout_rate)
+        self.linear = nn.Linear(input_dim, num_slot_labels)
+        
+    def forward(self, x):
+        x = self.dropout(x)
+        return self.linear(x)
+
+class BertIDSF(BertPreTrainedModel):
+    def __init__(self, config, intent_label_lst, slot_label_lst, n_layers=1):
+        super().__init__(config)
+        self.num_intent_labels = len(intent_label_lst)
+        self.num_slot_labels = len(slot_label_lst)
+        self.bert = BertModel(config=config)
+        
+        # Store dictionaries in config for later use
+        self.config.dict2 = {str(idx+1): label for idx, label in enumerate(slot_label_lst)}
+        self.config.inte2 = {str(idx+1): label for idx, label in enumerate(intent_label_lst)}
+        
+        classifier_dropout = (
+            config.classifier_dropout if config.classifier_dropout is not None else config.hidden_dropout_prob
+        )
+        self.dropout = nn.Dropout(classifier_dropout)
+        self.intent_classifier = IntentClassifier(config.hidden_size, self.num_intent_labels)
+        self.slot_classifier = SlotClassifier(config.hidden_size, self.num_slot_labels)
+
+    def forward(
+            self,
+            input_ids=None,
+            attention_mask=None,
+            token_type_ids=None,
+            position_ids=None,
+            head_mask=None,
+            inputs_embeds=None,
+            labels=None,
+            intents=None,
+            output_attentions=True,
+            lens=None,
+            device=None
+    ):
+        outputs = self.bert(
+            input_ids,
+            attention_mask=attention_mask,
+            token_type_ids=token_type_ids,
+            position_ids=position_ids,
+            head_mask=head_mask,
+            inputs_embeds=inputs_embeds,
+            output_attentions=True
+        )
+        
+        sequence_output = outputs[0]
+        sequence_output = self.dropout(sequence_output)
+        
+        intent_logits = self.intent_classifier(sequence_output[:, 0, :])
+        slot_logits = self.slot_classifier(sequence_output)
+        
+        total_loss = 0
+        
+        # Intent Softmax
+        if intents is not None:
+            intent_loss_fct = nn.CrossEntropyLoss()
+            intent_loss = intent_loss_fct(intent_logits.view(-1, self.num_intent_labels), intents.view(-1))
+            total_loss += 0.5 * intent_loss
+            
+        # Slot Softmax
+        if labels is not None:
+            slot_loss_fct = nn.CrossEntropyLoss(ignore_index=0)
+            # Only keep active parts of the loss
+            if attention_mask is not None:
+                active_loss = attention_mask.view(-1) == 1
+                active_logits = slot_logits.view(-1, self.num_slot_labels)[active_loss]
+                active_labels = labels.view(-1)[active_loss]
+                slot_loss = slot_loss_fct(active_logits, active_labels)
+            else:
+                slot_loss = slot_loss_fct(slot_logits.view(-1, self.num_slot_labels), labels.view(-1))
+            total_loss += 0.5 * slot_loss
+            
+        outputs = ((intent_logits, slot_logits),) + outputs[2:]  # add hidden states and attention if they are here
+        outputs = (total_loss,) + outputs
+        
+        return outputs  # (loss), scores, (hidden_states), (attentions)  # Logits is a tuple of intent and slot logits