first commit

.gitattributes

@@ -25,3 +25,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zstandard filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,51 @@
+---
+license: mit
+tags:
+- generated_from_trainer
+model-index:
+- name: mdeberta-wl-base-es
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# mdeberta-wl-base-es
+
+This model is a fine-tuned version of [microsoft/mdeberta-v3-base](https://huggingface.co/microsoft/mdeberta-v3-base) on an unknown dataset.
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 5e-05
+- train_batch_size: 16
+- eval_batch_size: 16
+- seed: 42
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: linear
+- num_epochs: 3.0
+
+### Training results
+
+
+
+### Framework versions
+
+- Transformers 4.21.0.dev0
+- Pytorch 1.11.0+cu113
+- Datasets 2.3.3.dev0
+- Tokenizers 0.12.1

added_tokens.json

@@ -0,0 +1,3 @@
+{
+  "[MASK]": 250101
+}

all_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 3.0,
+    "train_loss": 0.7381347277268193,
+    "train_runtime": 18401.3799,
+    "train_samples": 298465,
+    "train_samples_per_second": 48.659,
+    "train_steps_per_second": 3.041
+}

config.json

@@ -0,0 +1,35 @@
+{
+  "_name_or_path": "microsoft/mdeberta-v3-base",
+  "architectures": [
+    "DebertaV2ForMaskedLM"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "layer_norm_eps": 1e-07,
+  "max_position_embeddings": 512,
+  "max_relative_positions": -1,
+  "model_type": "deberta-v2",
+  "norm_rel_ebd": "layer_norm",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "pad_token_id": 0,
+  "pooler_dropout": 0,
+  "pooler_hidden_act": "gelu",
+  "pooler_hidden_size": 768,
+  "pos_att_type": [
+    "p2c",
+    "c2p"
+  ],
+  "position_biased_input": false,
+  "position_buckets": 256,
+  "relative_attention": true,
+  "share_att_key": true,
+  "torch_dtype": "float32",
+  "transformers_version": "4.21.0.dev0",
+  "type_vocab_size": 0,
+  "vocab_size": 250102
+}

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:57d32f66c0b9da878234d132e8bda518e43cf2cc72879b9142da7da0601f8a9b
+size 1113558185

special_tokens_map.json

@@ -0,0 +1,9 @@
+{
+  "bos_token": "[CLS]",
+  "cls_token": "[CLS]",
+  "eos_token": "[SEP]",
+  "mask_token": "[MASK]",
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "unk_token": "[UNK]"
+}

spm.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:13c8d666d62a7bc4ac8f040aab68e942c861f93303156cc28f5c7e885d86d6e3
+size 4305025

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6f06fc3bbcbbc8f07be861b9a6a69e177247bc549b2bcbf0483e4dd98c06b6f6
+size 16331301

tokenizer_config.json

@@ -0,0 +1,16 @@
+{
+  "bos_token": "[CLS]",
+  "cls_token": "[CLS]",
+  "do_lower_case": false,
+  "eos_token": "[SEP]",
+  "mask_token": "[MASK]",
+  "name_or_path": "microsoft/mdeberta-v3-base",
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "sp_model_kwargs": {},
+  "special_tokens_map_file": null,
+  "split_by_punct": false,
+  "tokenizer_class": "DebertaV2Tokenizer",
+  "unk_token": "[UNK]",
+  "vocab_type": "spm"
+}

train_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 3.0,
+    "train_loss": 0.7381347277268193,
+    "train_runtime": 18401.3799,
+    "train_samples": 298465,
+    "train_samples_per_second": 48.659,
+    "train_steps_per_second": 3.041
+}

trainer_state.json

@@ -0,0 +1,691 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 3.0,
+  "global_step": 55965,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.03,
+      "learning_rate": 4.955329223621907e-05,
+      "loss": 5.209,
+      "step": 500
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 4.910658447243813e-05,
+      "loss": 2.5487,
+      "step": 1000
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 4.86598767086572e-05,
+      "loss": 1.9748,
+      "step": 1500
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 4.8213168944876264e-05,
+      "loss": 1.7091,
+      "step": 2000
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 4.776646118109533e-05,
+      "loss": 1.5242,
+      "step": 2500
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 4.7319753417314395e-05,
+      "loss": 1.3854,
+      "step": 3000
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 4.687304565353346e-05,
+      "loss": 1.2987,
+      "step": 3500
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 4.6426337889752526e-05,
+      "loss": 1.2104,
+      "step": 4000
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 4.597963012597159e-05,
+      "loss": 1.1557,
+      "step": 4500
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 4.553292236219066e-05,
+      "loss": 1.1056,
+      "step": 5000
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 4.508621459840972e-05,
+      "loss": 1.0583,
+      "step": 5500
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 4.463950683462879e-05,
+      "loss": 1.0108,
+      "step": 6000
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 4.4192799070847854e-05,
+      "loss": 0.9883,
+      "step": 6500
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 4.374609130706692e-05,
+      "loss": 0.9585,
+      "step": 7000
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 4.3299383543285985e-05,
+      "loss": 0.9387,
+      "step": 7500
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 4.285267577950505e-05,
+      "loss": 0.9223,
+      "step": 8000
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 4.2405968015724116e-05,
+      "loss": 0.8995,
+      "step": 8500
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 4.195926025194318e-05,
+      "loss": 0.8792,
+      "step": 9000
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.151255248816225e-05,
+      "loss": 0.8591,
+      "step": 9500
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.106584472438131e-05,
+      "loss": 0.8346,
+      "step": 10000
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.061913696060038e-05,
+      "loss": 0.8202,
+      "step": 10500
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.017242919681944e-05,
+      "loss": 0.8039,
+      "step": 11000
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.972572143303851e-05,
+      "loss": 0.8032,
+      "step": 11500
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.9279013669257574e-05,
+      "loss": 0.7879,
+      "step": 12000
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.883230590547664e-05,
+      "loss": 0.7838,
+      "step": 12500
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 3.8385598141695705e-05,
+      "loss": 0.7634,
+      "step": 13000
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 3.793889037791477e-05,
+      "loss": 0.7533,
+      "step": 13500
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 3.7492182614133836e-05,
+      "loss": 0.7453,
+      "step": 14000
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 3.70454748503529e-05,
+      "loss": 0.7318,
+      "step": 14500
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 3.659876708657197e-05,
+      "loss": 0.7229,
+      "step": 15000
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 3.615205932279103e-05,
+      "loss": 0.7203,
+      "step": 15500
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 3.57053515590101e-05,
+      "loss": 0.7136,
+      "step": 16000
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 3.5258643795229163e-05,
+      "loss": 0.7046,
+      "step": 16500
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 3.481193603144823e-05,
+      "loss": 0.6975,
+      "step": 17000
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 3.4365228267667294e-05,
+      "loss": 0.6831,
+      "step": 17500
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 3.391852050388636e-05,
+      "loss": 0.679,
+      "step": 18000
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 3.3471812740105425e-05,
+      "loss": 0.6761,
+      "step": 18500
+    },
+    {
+      "epoch": 1.02,
+      "learning_rate": 3.302510497632449e-05,
+      "loss": 0.6676,
+      "step": 19000
+    },
+    {
+      "epoch": 1.05,
+      "learning_rate": 3.2578397212543556e-05,
+      "loss": 0.6622,
+      "step": 19500
+    },
+    {
+      "epoch": 1.07,
+      "learning_rate": 3.213168944876262e-05,
+      "loss": 0.6532,
+      "step": 20000
+    },
+    {
+      "epoch": 1.1,
+      "learning_rate": 3.168498168498169e-05,
+      "loss": 0.6475,
+      "step": 20500
+    },
+    {
+      "epoch": 1.13,
+      "learning_rate": 3.123827392120075e-05,
+      "loss": 0.6532,
+      "step": 21000
+    },
+    {
+      "epoch": 1.15,
+      "learning_rate": 3.079156615741982e-05,
+      "loss": 0.6462,
+      "step": 21500
+    },
+    {
+      "epoch": 1.18,
+      "learning_rate": 3.034485839363888e-05,
+      "loss": 0.6415,
+      "step": 22000
+    },
+    {
+      "epoch": 1.21,
+      "learning_rate": 2.9898150629857946e-05,
+      "loss": 0.6298,
+      "step": 22500
+    },
+    {
+      "epoch": 1.23,
+      "learning_rate": 2.945144286607701e-05,
+      "loss": 0.6252,
+      "step": 23000
+    },
+    {
+      "epoch": 1.26,
+      "learning_rate": 2.9004735102296077e-05,
+      "loss": 0.6248,
+      "step": 23500
+    },
+    {
+      "epoch": 1.29,
+      "learning_rate": 2.8558027338515142e-05,
+      "loss": 0.616,
+      "step": 24000
+    },
+    {
+      "epoch": 1.31,
+      "learning_rate": 2.811131957473421e-05,
+      "loss": 0.6198,
+      "step": 24500
+    },
+    {
+      "epoch": 1.34,
+      "learning_rate": 2.7664611810953277e-05,
+      "loss": 0.6107,
+      "step": 25000
+    },
+    {
+      "epoch": 1.37,
+      "learning_rate": 2.7217904047172342e-05,
+      "loss": 0.6011,
+      "step": 25500
+    },
+    {
+      "epoch": 1.39,
+      "learning_rate": 2.6771196283391408e-05,
+      "loss": 0.6104,
+      "step": 26000
+    },
+    {
+      "epoch": 1.42,
+      "learning_rate": 2.6324488519610473e-05,
+      "loss": 0.6004,
+      "step": 26500
+    },
+    {
+      "epoch": 1.45,
+      "learning_rate": 2.587778075582954e-05,
+      "loss": 0.5984,
+      "step": 27000
+    },
+    {
+      "epoch": 1.47,
+      "learning_rate": 2.5431072992048604e-05,
+      "loss": 0.5893,
+      "step": 27500
+    },
+    {
+      "epoch": 1.5,
+      "learning_rate": 2.4984365228267666e-05,
+      "loss": 0.5958,
+      "step": 28000
+    },
+    {
+      "epoch": 1.53,
+      "learning_rate": 2.4537657464486735e-05,
+      "loss": 0.5875,
+      "step": 28500
+    },
+    {
+      "epoch": 1.55,
+      "learning_rate": 2.40909497007058e-05,
+      "loss": 0.5763,
+      "step": 29000
+    },
+    {
+      "epoch": 1.58,
+      "learning_rate": 2.3644241936924866e-05,
+      "loss": 0.5839,
+      "step": 29500
+    },
+    {
+      "epoch": 1.61,
+      "learning_rate": 2.319753417314393e-05,
+      "loss": 0.5802,
+      "step": 30000
+    },
+    {
+      "epoch": 1.63,
+      "learning_rate": 2.2750826409362997e-05,
+      "loss": 0.5668,
+      "step": 30500
+    },
+    {
+      "epoch": 1.66,
+      "learning_rate": 2.2304118645582063e-05,
+      "loss": 0.5825,
+      "step": 31000
+    },
+    {
+      "epoch": 1.69,
+      "learning_rate": 2.1857410881801128e-05,
+      "loss": 0.5746,
+      "step": 31500
+    },
+    {
+      "epoch": 1.72,
+      "learning_rate": 2.1410703118020194e-05,
+      "loss": 0.5668,
+      "step": 32000
+    },
+    {
+      "epoch": 1.74,
+      "learning_rate": 2.096399535423926e-05,
+      "loss": 0.5612,
+      "step": 32500
+    },
+    {
+      "epoch": 1.77,
+      "learning_rate": 2.051728759045832e-05,
+      "loss": 0.5643,
+      "step": 33000
+    },
+    {
+      "epoch": 1.8,
+      "learning_rate": 2.0070579826677387e-05,
+      "loss": 0.5555,
+      "step": 33500
+    },
+    {
+      "epoch": 1.82,
+      "learning_rate": 1.9623872062896452e-05,
+      "loss": 0.558,
+      "step": 34000
+    },
+    {
+      "epoch": 1.85,
+      "learning_rate": 1.9177164299115518e-05,
+      "loss": 0.556,
+      "step": 34500
+    },
+    {
+      "epoch": 1.88,
+      "learning_rate": 1.8730456535334583e-05,
+      "loss": 0.5487,
+      "step": 35000
+    },
+    {
+      "epoch": 1.9,
+      "learning_rate": 1.828374877155365e-05,
+      "loss": 0.547,
+      "step": 35500
+    },
+    {
+      "epoch": 1.93,
+      "learning_rate": 1.7837041007772714e-05,
+      "loss": 0.5568,
+      "step": 36000
+    },
+    {
+      "epoch": 1.96,
+      "learning_rate": 1.7390333243991783e-05,
+      "loss": 0.545,
+      "step": 36500
+    },
+    {
+      "epoch": 1.98,
+      "learning_rate": 1.694362548021085e-05,
+      "loss": 0.5455,
+      "step": 37000
+    },
+    {
+      "epoch": 2.01,
+      "learning_rate": 1.6496917716429914e-05,
+      "loss": 0.5455,
+      "step": 37500
+    },
+    {
+      "epoch": 2.04,
+      "learning_rate": 1.605020995264898e-05,
+      "loss": 0.5358,
+      "step": 38000
+    },
+    {
+      "epoch": 2.06,
+      "learning_rate": 1.5603502188868045e-05,
+      "loss": 0.5383,
+      "step": 38500
+    },
+    {
+      "epoch": 2.09,
+      "learning_rate": 1.5156794425087109e-05,
+      "loss": 0.5345,
+      "step": 39000
+    },
+    {
+      "epoch": 2.12,
+      "learning_rate": 1.4710086661306174e-05,
+      "loss": 0.5295,
+      "step": 39500
+    },
+    {
+      "epoch": 2.14,
+      "learning_rate": 1.426337889752524e-05,
+      "loss": 0.5278,
+      "step": 40000
+    },
+    {
+      "epoch": 2.17,
+      "learning_rate": 1.3816671133744305e-05,
+      "loss": 0.5343,
+      "step": 40500
+    },
+    {
+      "epoch": 2.2,
+      "learning_rate": 1.336996336996337e-05,
+      "loss": 0.5325,
+      "step": 41000
+    },
+    {
+      "epoch": 2.22,
+      "learning_rate": 1.2923255606182436e-05,
+      "loss": 0.5218,
+      "step": 41500
+    },
+    {
+      "epoch": 2.25,
+      "learning_rate": 1.2476547842401502e-05,
+      "loss": 0.5282,
+      "step": 42000
+    },
+    {
+      "epoch": 2.28,
+      "learning_rate": 1.2029840078620567e-05,
+      "loss": 0.5183,
+      "step": 42500
+    },
+    {
+      "epoch": 2.31,
+      "learning_rate": 1.1583132314839633e-05,
+      "loss": 0.5152,
+      "step": 43000
+    },
+    {
+      "epoch": 2.33,
+      "learning_rate": 1.1136424551058698e-05,
+      "loss": 0.5245,
+      "step": 43500
+    },
+    {
+      "epoch": 2.36,
+      "learning_rate": 1.0689716787277764e-05,
+      "loss": 0.5272,
+      "step": 44000
+    },
+    {
+      "epoch": 2.39,
+      "learning_rate": 1.0243009023496829e-05,
+      "loss": 0.5229,
+      "step": 44500
+    },
+    {
+      "epoch": 2.41,
+      "learning_rate": 9.796301259715895e-06,
+      "loss": 0.5212,
+      "step": 45000
+    },
+    {
+      "epoch": 2.44,
+      "learning_rate": 9.34959349593496e-06,
+      "loss": 0.5198,
+      "step": 45500
+    },
+    {
+      "epoch": 2.47,
+      "learning_rate": 8.902885732154024e-06,
+      "loss": 0.5073,
+      "step": 46000
+    },
+    {
+      "epoch": 2.49,
+      "learning_rate": 8.456177968373091e-06,
+      "loss": 0.5061,
+      "step": 46500
+    },
+    {
+      "epoch": 2.52,
+      "learning_rate": 8.009470204592157e-06,
+      "loss": 0.5164,
+      "step": 47000
+    },
+    {
+      "epoch": 2.55,
+      "learning_rate": 7.562762440811222e-06,
+      "loss": 0.5046,
+      "step": 47500
+    },
+    {
+      "epoch": 2.57,
+      "learning_rate": 7.1160546770302875e-06,
+      "loss": 0.5089,
+      "step": 48000
+    },
+    {
+      "epoch": 2.6,
+      "learning_rate": 6.669346913249352e-06,
+      "loss": 0.5097,
+      "step": 48500
+    },
+    {
+      "epoch": 2.63,
+      "learning_rate": 6.222639149468418e-06,
+      "loss": 0.5037,
+      "step": 49000
+    },
+    {
+      "epoch": 2.65,
+      "learning_rate": 5.775931385687484e-06,
+      "loss": 0.5115,
+      "step": 49500
+    },
+    {
+      "epoch": 2.68,
+      "learning_rate": 5.329223621906549e-06,
+      "loss": 0.5096,
+      "step": 50000
+    },
+    {
+      "epoch": 2.71,
+      "learning_rate": 4.882515858125614e-06,
+      "loss": 0.4986,
+      "step": 50500
+    },
+    {
+      "epoch": 2.73,
+      "learning_rate": 4.43580809434468e-06,
+      "loss": 0.4927,
+      "step": 51000
+    },
+    {
+      "epoch": 2.76,
+      "learning_rate": 3.989100330563746e-06,
+      "loss": 0.51,
+      "step": 51500
+    },
+    {
+      "epoch": 2.79,
+      "learning_rate": 3.542392566782811e-06,
+      "loss": 0.5061,
+      "step": 52000
+    },
+    {
+      "epoch": 2.81,
+      "learning_rate": 3.095684803001876e-06,
+      "loss": 0.5024,
+      "step": 52500
+    },
+    {
+      "epoch": 2.84,
+      "learning_rate": 2.648977039220942e-06,
+      "loss": 0.4951,
+      "step": 53000
+    },
+    {
+      "epoch": 2.87,
+      "learning_rate": 2.202269275440007e-06,
+      "loss": 0.4984,
+      "step": 53500
+    },
+    {
+      "epoch": 2.89,
+      "learning_rate": 1.7555615116590727e-06,
+      "loss": 0.497,
+      "step": 54000
+    },
+    {
+      "epoch": 2.92,
+      "learning_rate": 1.3088537478781382e-06,
+      "loss": 0.5004,
+      "step": 54500
+    },
+    {
+      "epoch": 2.95,
+      "learning_rate": 8.621459840972037e-07,
+      "loss": 0.4959,
+      "step": 55000
+    },
+    {
+      "epoch": 2.98,
+      "learning_rate": 4.1543822031626915e-07,
+      "loss": 0.4943,
+      "step": 55500
+    },
+    {
+      "epoch": 3.0,
+      "step": 55965,
+      "total_flos": 2.3628048684309504e+17,
+      "train_loss": 0.7381347277268193,
+      "train_runtime": 18401.3799,
+      "train_samples_per_second": 48.659,
+      "train_steps_per_second": 3.041
+    }
+  ],
+  "max_steps": 55965,
+  "num_train_epochs": 3,
+  "total_flos": 2.3628048684309504e+17,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4cd05523275a552b097227da22fea8b685f7315829e8eb9245854d6202afde81
+size 3311