Upload custom DeBERTa-LSTM-Attention model

config.json

@@ -0,0 +1,32 @@
+{
+  "attention_probs_dropout_prob": 0.1,
+  "auto_map": {
+    "AutoModel": "model.DeBERTaLSTMClassifier",
+    "AutoModelForSequenceClassification": "model.DeBERTaLSTMClassifier"
+  },
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "layer_norm_eps": 1e-07,
+  "legacy": true,
+  "max_position_embeddings": 512,
+  "max_relative_positions": -1,
+  "model_type": "deberta",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "pad_token_id": 0,
+  "pooler_dropout": 0,
+  "pooler_hidden_act": "gelu",
+  "pooler_hidden_size": 768,
+  "pos_att_type": [
+    "c2p",
+    "p2c"
+  ],
+  "position_biased_input": false,
+  "relative_attention": true,
+  "transformers_version": "4.57.2",
+  "type_vocab_size": 0,
+  "vocab_size": 50265
+}

model.py

@@ -0,0 +1,49 @@
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from transformers import AutoModel
+
+class DeBERTaLSTMClassifier(nn.Module):
+    def __init__(self, hidden_dim=128, num_labels=2):
+        super().__init__()
+
+        self.deberta = AutoModel.from_pretrained("microsoft/deberta-base")
+        for param in self.deberta.parameters():
+            param.requires_grad = False  # freeze DeBERTa (as we don't have enough resources, we will not train DeBERTa in this model)
+
+        self.lstm = nn.LSTM(
+            input_size=self.deberta.config.hidden_size,
+            hidden_size=hidden_dim,
+            batch_first=True,
+            bidirectional=True
+        )
+
+        self.fc = nn.Linear(hidden_dim * 2, num_labels)
+        
+        # Attention layer để tính token importance
+        self.attention = nn.Linear(hidden_dim * 2, 1)
+
+    def forward(self, input_ids, attention_mask, return_attention=False):
+        with torch.no_grad():
+            outputs = self.deberta(input_ids=input_ids, attention_mask=attention_mask, output_attentions=True)
+
+        lstm_out, _ = self.lstm(outputs.last_hidden_state)  # shape: [batch, seq_len, hidden*2]
+        
+        if return_attention:
+            # Tính attention weights cho từng token
+            attention_weights = self.attention(lstm_out)  # [batch, seq_len, 1]
+            attention_weights = F.softmax(attention_weights.squeeze(-1), dim=-1)  # [batch, seq_len]
+            
+            # Apply attention mask
+            attention_weights = attention_weights * attention_mask.float()
+            attention_weights = attention_weights / (attention_weights.sum(dim=-1, keepdim=True) + 1e-8)
+            
+            # Weighted sum of LSTM outputs
+            attended_output = torch.sum(lstm_out * attention_weights.unsqueeze(-1), dim=1)
+            logits = self.fc(attended_output)
+            
+            return logits, attention_weights, outputs.attentions
+        else:
+            final_hidden = lstm_out[:, -1, :]  # last token output
+            logits = self.fc(final_hidden)
+            return logits

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ea43b73d419c4314ca74e4d921e6b4bfefa6ed4c6f5805f35720b5b26a365035
+size 558172963

special_tokens_map.json

@@ -0,0 +1,51 @@
+{
+  "bos_token": {
+    "content": "[CLS]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "cls_token": {
+    "content": "[CLS]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "[SEP]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "mask_token": {
+    "content": "[MASK]",
+    "lstrip": true,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "[PAD]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "sep_token": {
+    "content": "[SEP]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "[UNK]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer_config.json

@@ -0,0 +1,60 @@
+{
+  "add_bos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "[PAD]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "[CLS]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "[SEP]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "[UNK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50264": {
+      "content": "[MASK]",
+      "lstrip": true,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "[CLS]",
+  "clean_up_tokenization_spaces": false,
+  "cls_token": "[CLS]",
+  "do_lower_case": false,
+  "eos_token": "[SEP]",
+  "errors": "replace",
+  "extra_special_tokens": {},
+  "mask_token": "[MASK]",
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "tokenizer_class": "DebertaTokenizer",
+  "unk_token": "[UNK]",
+  "vocab_type": "gpt2"
+}