deberta_essay

README.md

@@ -1,8 +1,8 @@
 ---
-license: apache-2.0
+license: mit
+base_model: microsoft/deberta-v3-base
 tags:
 - generated_from_trainer
-base_model: distilbert/distilbert-base-uncased
 metrics:
 - accuracy
 model-index:
@@ -15,13 +15,13 @@ should probably proofread and complete it, then remove this comment. -->
 
 # bert_essay
 
-This model is a fine-tuned version of [distilbert/distilbert-base-uncased](https://huggingface.co/distilbert/distilbert-base-uncased) on the None dataset.
+This model is a fine-tuned version of [microsoft/deberta-v3-base](https://huggingface.co/microsoft/deberta-v3-base) on the None dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.3554
-- Mse: 0.3554
-- Mae: 0.4632
-- R2: 0.6556
-- Accuracy: 0.2821
+- Loss: 0.3763
+- Mse: 0.3763
+- Mae: 0.4747
+- R2: 0.6434
+- Accuracy: 0.2684
 
 ## Model description
 
@@ -41,8 +41,8 @@ More information needed
 
 The following hyperparameters were used during training:
 - learning_rate: 2e-05
-- train_batch_size: 32
-- eval_batch_size: 32
+- train_batch_size: 16
+- eval_batch_size: 16
 - seed: 42
 - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
 - lr_scheduler_type: linear
@@ -52,9 +52,9 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch | Step | Validation Loss | Mse    | Mae    | R2     | Accuracy |
 |:-------------:|:-----:|:----:|:---------------:|:------:|:------:|:------:|:--------:|
-| No log        | 1.0   | 433  | 0.3749          | 0.3749 | 0.4772 | 0.6367 | 0.2992   |
-| 0.6371        | 2.0   | 866  | 0.3572          | 0.3572 | 0.4653 | 0.6539 | 0.2848   |
-| 0.3565        | 3.0   | 1299 | 0.3554          | 0.3554 | 0.4632 | 0.6556 | 0.2821   |
+| 0.6577        | 1.0   | 866  | 0.5250          | 0.5250 | 0.5685 | 0.5025 | 0.2674   |
+| 0.3355        | 2.0   | 1732 | 0.4174          | 0.4174 | 0.5027 | 0.6045 | 0.2615   |
+| 0.2592        | 3.0   | 2598 | 0.3763          | 0.3763 | 0.4747 | 0.6434 | 0.2684   |
 
 
 ### Framework versions

config.json

@@ -1,30 +1,41 @@
 {
-  "_name_or_path": "distilbert/distilbert-base-uncased",
-  "activation": "gelu",
+  "_name_or_path": "microsoft/deberta-v3-base",
   "architectures": [
-    "DistilBertForSequenceClassification"
+    "DebertaV2ForSequenceClassification"
   ],
-  "attention_dropout": 0.1,
-  "dim": 768,
-  "dropout": 0.1,
-  "hidden_dim": 3072,
+  "attention_probs_dropout_prob": 0.1,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
   "id2label": {
     "0": "LABEL_0"
   },
   "initializer_range": 0.02,
+  "intermediate_size": 3072,
   "label2id": {
     "LABEL_0": 0
   },
+  "layer_norm_eps": 1e-07,
   "max_position_embeddings": 512,
-  "model_type": "distilbert",
-  "n_heads": 12,
-  "n_layers": 6,
+  "max_relative_positions": -1,
+  "model_type": "deberta-v2",
+  "norm_rel_ebd": "layer_norm",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
   "pad_token_id": 0,
-  "qa_dropout": 0.1,
-  "seq_classif_dropout": 0.2,
-  "sinusoidal_pos_embds": false,
-  "tie_weights_": true,
+  "pooler_dropout": 0,
+  "pooler_hidden_act": "gelu",
+  "pooler_hidden_size": 768,
+  "pos_att_type": [
+    "p2c",
+    "c2p"
+  ],
+  "position_biased_input": false,
+  "position_buckets": 256,
+  "relative_attention": true,
+  "share_att_key": true,
   "torch_dtype": "float32",
   "transformers_version": "4.39.3",
-  "vocab_size": 30522
+  "type_vocab_size": 0,
+  "vocab_size": 128100
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e7fc9771802e9fabc523592cc0d79507c228ef64923630eee4baecbce837821b
-size 267829484
+oid sha256:c1812bf754782abf307c8f5900818f142a32a72397452e7bf9ca404512f83b69
+size 737716196

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:209747ef623254006b35addf4eebc9abaaa87ff1b8058272a37de4143f6ea566
+oid sha256:62ca48d0e52da4a600529ac47144f2d73bf8de100838c034db62ddf3fdce34a7
 size 4920