feat: add custom BERT architecture and weights to safetensors

configuration_veritas.py

@@ -0,0 +1,14 @@
+from transformers import PretrainedConfig
+
+class BERTClassifierConfig(PretrainedConfig):
+    model_type = "bert_classifier"
+
+    def __init__(
+        self,
+        drop_rate: float = 0.2,
+        num_labels: int = 2,
+        **kwargs
+    ):
+        super().__init__(**kwargs)
+        self.drop_rate = drop_rate
+        self.num_labels = num_labels

gitattributes

@@ -0,0 +1,36 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+veritas-lstm-ptbr.keras filter=lfs diff=lfs merge=lfs -text

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:493e939db2633b6a6954313800591d44bf0e0597981a10dded8aedf187d395f7
+size 435817112

modeling_veritas.py

@@ -0,0 +1,64 @@
+import torch
+import torch.nn as nn
+
+from transformers import (
+    AutoModel,
+    PreTrainedModel
+)
+from transformers.modeling_outputs import SequenceClassifierOutput
+
+from .configuration_veritas import BERTClassifierConfig
+
+from transformers import AutoModel, AutoConfig
+
+class BERTClassifier(PreTrainedModel):
+    config_class = BERTClassifierConfig
+
+    def __init__(self, config):
+        super().__init__(config)
+
+        bert_config = AutoConfig.from_pretrained(
+            "neuralmind/bert-base-portuguese-cased"
+        )
+
+        self.bert = AutoModel.from_config(bert_config)
+
+        for param in self.bert.encoder.layer[:6].parameters():
+            param.requires_grad = False
+
+        hidden = self.bert.config.hidden_size
+
+        self.classifier = nn.Sequential(
+            nn.Linear(hidden, 32),
+            nn.GELU(),
+            nn.Dropout(config.drop_rate),
+            nn.Linear(32, 16),
+            nn.GELU(),
+            nn.Linear(16, config.num_labels)
+        )
+
+        self.post_init()
+
+    def forward(
+        self,
+        input_ids=None,
+        attention_mask=None,
+        labels=None
+    ):
+        outputs = self.bert(
+            input_ids=input_ids,
+            attention_mask=attention_mask
+        )
+
+        pooler = outputs.last_hidden_state[:, 0, :]
+        logits = self.classifier(pooler)
+
+        loss = None
+        if labels is not None:
+            loss_fn = nn.CrossEntropyLoss()
+            loss = loss_fn(logits, labels)
+
+        return SequenceClassifierOutput(
+            loss=loss,
+            logits=logits
+        )

tokenizer_config.json

@@ -0,0 +1,14 @@
+{
+  "backend": "tokenizers",
+  "cls_token": "[CLS]",
+  "do_lower_case": false,
+  "is_local": false,
+  "mask_token": "[MASK]",
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "strip_accents": null,
+  "tokenize_chinese_chars": true,
+  "tokenizer_class": "BertTokenizer",
+  "unk_token": "[UNK]"
+}