Add model definition and pytorch model

.gitattributes

@@ -14,3 +14,4 @@
 *.pb filter=lfs diff=lfs merge=lfs -text
 *.pt filter=lfs diff=lfs merge=lfs -text
 *.pth filter=lfs diff=lfs merge=lfs -text
+model/pytorch_model.bin filter=lfs diff=lfs merge=lfs -text

README.md

@@ -21,10 +21,9 @@ news articles in English. It was introduced in
 
 ## Model description
 
-
-
-## Intended users & limitations
-
+Please find propaganda definition here:
+https://propaganda.qcri.org/annotations/definitions.html
 
 
 ## How to use
+

model.py

@@ -0,0 +1,128 @@
+__author__ = "Yifan Zhang (yzhang@hbku.edu.qa)"
+__copyright__ = "Copyright (C) 2021, Qatar Computing Research Institute, HBKU, Doha"
+
+
+from dataclasses import dataclass
+from typing import Optional, Tuple
+import torch
+from torch import nn
+from torch.nn.functional import sigmoid
+from transformers import BertPreTrainedModel, BertModel
+from transformers.file_utils import ModelOutput
+
+
+TOKEN_TAGS = (
+    "<PAD>", "O", 
+    "Name_Calling,Labeling", "Repetition", "Slogans", "Appeal_to_fear-prejudice", "Doubt",
+    "Exaggeration,Minimisation", "Flag-Waving", "Loaded_Language",
+    "Reductio_ad_hitlerum", "Bandwagon",
+    "Causal_Oversimplification", "Obfuscation,Intentional_Vagueness,Confusion", "Appeal_to_Authority", "Black-and-White_Fallacy",
+    "Thought-terminating_Cliches", "Red_Herring", "Straw_Men", "Whataboutism"
+)
+
+
+SEQUENCE_TAGS = ("Non-prop", "Prop")
+
+
+@dataclass
+class TokenAndSequenceJointClassifierOutput(ModelOutput):
+    loss: Optional[torch.FloatTensor] = None
+    token_logits: torch.FloatTensor = None
+    sequence_logits: torch.FloatTensor = None
+    hidden_states: Optional[Tuple[torch.FloatTensor]] = None
+    attentions: Optional[Tuple[torch.FloatTensor]] = None
+
+
+class BertForTokenAndSequenceJointClassification(BertPreTrainedModel):
+
+    def __init__(self, config):
+        super().__init__(config)
+        self.num_token_labels = 20
+        self.num_sequence_labels = 2
+
+        self.token_tags = TOKEN_TAGS
+        self.sequence_tags = SEQUENCE_TAGS 
+
+        self.alpha = 0.9
+
+        self.bert = BertModel(config)
+        self.dropout = nn.Dropout(config.hidden_dropout_prob)
+        self.classifier = nn.ModuleList([
+            nn.Linear(config.hidden_size, self.num_token_labels),
+            nn.Linear(config.hidden_size, self.num_sequence_labels),
+        ])
+        self.masking_gate = nn.Linear(2, 1)
+
+        self.init_weights()
+        self.merge_classifier_1 = nn.Linear(self.num_token_labels + self.num_sequence_labels, self.num_token_labels)
+
+    def forward(
+        self,
+        input_ids=None,
+        attention_mask=None,
+        token_type_ids=None,
+        position_ids=None,
+        head_mask=None,
+        inputs_embeds=None,
+        labels=None,
+        output_attentions=None,
+        output_hidden_states=None,
+        return_dict=True,
+    ):
+        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+
+        outputs = self.bert(
+            input_ids,
+            attention_mask=attention_mask,
+            token_type_ids=token_type_ids,
+            position_ids=position_ids,
+            head_mask=head_mask,
+            inputs_embeds=inputs_embeds,
+            output_attentions=output_attentions,
+            output_hidden_states=output_hidden_states,
+        )
+
+        sequence_output = outputs[0]
+        pooler_output = outputs[1]
+
+        sequence_output = self.dropout(sequence_output)
+        token_logits = self.classifier[0](sequence_output)
+
+        pooler_output = self.dropout(pooler_output)
+        sequence_logits = self.classifier[1](pooler_output)
+
+        gate = torch.sigmoid(self.masking_gate(sequence_logits))
+
+        gates = gate.unsqueeze(1).repeat(1, token_logits.size()[1], token_logits.size()[2])
+
+        weighted_token_logits = torch.mul(gates, token_logits)
+
+        logits = [weighted_token_logits, sequence_logits]
+
+        loss = None
+        if labels is not None:
+            criterion = nn.CrossEntropyLoss(ignore_index=0)
+            binary_criterion = nn.BCEWithLogitsLoss(pos_weight=torch.Tensor([3932/14263]).cuda())
+            loss_fct = CrossEntropyLoss()
+            weighted_token_logits = weighted_token_logits.view(-1, weighted_token_logits.shape[-1])
+            sequence_logits = sequence_logits.view(-1, sequence_logits.shape[-1])
+
+            token_loss = criterion(weighted_token_logits, labels)
+            sequence_label = torch.LongTensor([1] if any([label > 0 for label in labels]) else [0])
+            sequence_loss = binary_criterion(sequence_logits, sequence_label)
+
+            loss = self.alpha*loss[0] + (1-self.alpha)*loss[1]
+
+        if not return_dict:
+            output = (logits,) + outputs[2:]
+            return ((loss,) + output) if loss is not None else output
+
+        return TokenAndSequenceJointClassifierOutput(
+            loss=loss,
+            token_logits=weighted_token_logits,
+            sequence_logits=sequence_logits,
+            hidden_states=outputs.hidden_states,
+            attentions=outputs.attentions,
+        )
+
+

model/config.json

@@ -0,0 +1,25 @@
+{
+  "_name_or_path": "./model",
+  "architectures": [
+    "BertForTokenAndSequenceJointClassification"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "gradient_checkpointing": false,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "layer_norm_eps": 1e-12,
+  "max_position_embeddings": 512,
+  "model_type": "bert",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "output_hidden_states": true,
+  "pad_token_id": 0,
+  "position_embedding_type": "absolute",
+  "transformers_version": "4.5.1",
+  "type_vocab_size": 2,
+  "use_cache": true,
+  "vocab_size": 28996
+}

model/pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:92c6db6f7ede8bff314c1f6324f9650579012f12b2a2478ab458ff28694b48de
+size 433399567