Duplicate from microsoft/deberta-base

Co-authored-by: Julien Chaumond <julien-c@users.noreply.huggingface.co>

.gitattributes

@@ -0,0 +1,8 @@
+*.bin.* filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tar.gz filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,40 @@
+---
+language: en
+tags: 
+- deberta-v1
+- fill-mask
+thumbnail: https://huggingface.co/front/thumbnails/microsoft.png
+license: mit
+---
+
+## DeBERTa: Decoding-enhanced BERT with Disentangled Attention
+
+[DeBERTa](https://arxiv.org/abs/2006.03654) improves the BERT and RoBERTa models using disentangled attention and enhanced mask decoder. It outperforms BERT and RoBERTa on  majority of NLU tasks with 80GB training data. 
+
+Please check the [official repository](https://github.com/microsoft/DeBERTa) for more details and updates.
+
+
+#### Fine-tuning on NLU tasks
+
+We present the dev results on SQuAD 1.1/2.0 and MNLI tasks.
+
+| Model             | SQuAD 1.1 | SQuAD 2.0 | MNLI-m |
+|-------------------|-----------|-----------|--------|
+| RoBERTa-base      | 91.5/84.6 | 83.7/80.5 | 87.6   |
+| XLNet-Large       | -/-       | -/80.2    | 86.8   |
+| **DeBERTa-base**  | 93.1/87.2 | 86.2/83.1 | 88.8   |
+
+### Citation
+
+If you find DeBERTa useful for your work, please cite the following paper:
+
+``` latex
+@inproceedings{
+he2021deberta,
+title={DEBERTA: DECODING-ENHANCED BERT WITH DISENTANGLED ATTENTION},
+author={Pengcheng He and Xiaodong Liu and Jianfeng Gao and Weizhu Chen},
+booktitle={International Conference on Learning Representations},
+year={2021},
+url={https://openreview.net/forum?id=XPZIaotutsD}
+}
+```

bpe_encoder.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e7c6f9eecb461c01e09c00656ccf3e27944b9e74bfe29e51632b13d3cd9d6c8e
+size 3917897

config.json

@@ -0,0 +1,19 @@
+{
+	"model_type": "deberta",
+	"attention_probs_dropout_prob": 0.1,
+	"hidden_act": "gelu",
+	"hidden_dropout_prob": 0.1,
+	"hidden_size": 768,
+	"initializer_range": 0.02,
+	"intermediate_size": 3072,
+	"max_position_embeddings": 512,
+	"relative_attention": true,
+	"pos_att_type": "c2p|p2c",
+	"layer_norm_eps": 1e-7,
+	"max_relative_positions": -1,
+	"position_biased_input": false,
+	"num_attention_heads": 12,
+	"num_hidden_layers": 12,
+	"type_vocab_size": 0,
+	"vocab_size": 50265
+}

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:81ab61c5dc9111f81e528b7b3ae93997f4ae72ba531e60587835be894ab0ffe5
+size 558614189

rust_model.ot

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:42a8937bf8d3d98be44be18d92c0fafdebcddf839c8be2d72e18ec1b408b1e3c
+size 558613749

tf_model.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:16442fa0caaa8927e020e212b89b354db00a85f06c9b68eda20815233f619414
+size 554669920

tokenizer_config.json

@@ -0,0 +1,4 @@
+{
+  "do_lower_case": false,
+	"vocab_type": "gpt2"
+}