Upload 12 files

config.json

@@ -0,0 +1,39 @@
+{
+  "_name_or_path": "google/electra-large-discriminator",
+  "architectures": [
+    "ElectraForSequenceClassification"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "classifier_dropout": null,
+  "embedding_size": 1024,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 1024,
+  "id2label": {
+    "0": "NEGATIVE",
+    "1": "POSITIVE"
+  },
+  "initializer_range": 0.02,
+  "intermediate_size": 4096,
+  "label2id": {
+    "NEGATIVE": 0,
+    "POSITIVE": 1
+  },
+  "layer_norm_eps": 1e-12,
+  "max_position_embeddings": 512,
+  "model_type": "electra",
+  "num_attention_heads": 16,
+  "num_hidden_layers": 24,
+  "pad_token_id": 0,
+  "position_embedding_type": "absolute",
+  "problem_type": "single_label_classification",
+  "summary_activation": "gelu",
+  "summary_last_dropout": 0.1,
+  "summary_type": "first",
+  "summary_use_proj": true,
+  "torch_dtype": "float32",
+  "transformers_version": "4.31.0",
+  "type_vocab_size": 2,
+  "use_cache": true,
+  "vocab_size": 30522
+}

optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:585dfd8f3e2227c2e8d4f41107da6b7e73b07aeef18cb8dd1d39405923fb86b6
+size 2681382842

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:00663d985741fafa84eb89a974f02286257b7de9cd489b98771eb47d3d7e4915
+size 1340711790

readme.md

@@ -0,0 +1,14 @@
+`google/electra-large-discriminator` finetuned on imdb dataset for 2 epoches.
+
+Large examples tokenized with head and tail parts of a review, as described in [How to Fine-Tune BERT for Text Classification?](https://arxiv.org/abs/1905.05583)
+
+```python
+def preprocess_function(example):
+    tokens = tokenizer(example["text"], truncation=False)
+    if len(tokens['input_ids']) > 512:
+        tokens['input_ids'] = tokens['input_ids'][:129] + \
+            [102] + tokens['input_ids'][-382:]
+        tokens['token_type_ids'] = [0]*512
+        tokens['attention_mask'] = [1]*512
+    return tokens
+```

rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:53d2fecb476a681abf3740c1a0c5f92578590207d79e8495a51d786ab9386914
+size 14244

scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f59813ac2216377e393dcff7ff2c3c40f95f23641544178497cd91418272ee8d
+size 1064

special_tokens_map.json

@@ -0,0 +1,7 @@
+{
+  "cls_token": "[CLS]",
+  "mask_token": "[MASK]",
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "unk_token": "[UNK]"
+}

tokenizer_config.json

@@ -0,0 +1,13 @@
+{
+  "clean_up_tokenization_spaces": true,
+  "cls_token": "[CLS]",
+  "do_lower_case": true,
+  "mask_token": "[MASK]",
+  "model_max_length": 512,
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "strip_accents": null,
+  "tokenize_chinese_chars": true,
+  "tokenizer_class": "ElectraTokenizer",
+  "unk_token": "[UNK]"
+}

trainer_state.json

@@ -0,0 +1,124 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.99936,
+  "global_step": 1562,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.13,
+      "learning_rate": 2e-05,
+      "loss": 0.5028,
+      "step": 100
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 1.863201094391245e-05,
+      "loss": 0.2106,
+      "step": 200
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 1.72640218878249e-05,
+      "loss": 0.1829,
+      "step": 300
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 1.5896032831737347e-05,
+      "loss": 0.165,
+      "step": 400
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 1.4528043775649796e-05,
+      "loss": 0.1482,
+      "step": 500
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 1.3160054719562245e-05,
+      "loss": 0.1394,
+      "step": 600
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.1792065663474691e-05,
+      "loss": 0.1451,
+      "step": 700
+    },
+    {
+      "epoch": 1.0,
+      "eval_accuracy": 0.96384,
+      "eval_loss": 0.10218027979135513,
+      "eval_runtime": 754.598,
+      "eval_samples_per_second": 33.13,
+      "eval_steps_per_second": 4.141,
+      "step": 781
+    },
+    {
+      "epoch": 1.02,
+      "learning_rate": 1.0424076607387142e-05,
+      "loss": 0.1213,
+      "step": 800
+    },
+    {
+      "epoch": 1.15,
+      "learning_rate": 9.05608755129959e-06,
+      "loss": 0.091,
+      "step": 900
+    },
+    {
+      "epoch": 1.28,
+      "learning_rate": 7.688098495212039e-06,
+      "loss": 0.0988,
+      "step": 1000
+    },
+    {
+      "epoch": 1.41,
+      "learning_rate": 6.3201094391244875e-06,
+      "loss": 0.0822,
+      "step": 1100
+    },
+    {
+      "epoch": 1.54,
+      "learning_rate": 4.952120383036936e-06,
+      "loss": 0.092,
+      "step": 1200
+    },
+    {
+      "epoch": 1.66,
+      "learning_rate": 3.584131326949385e-06,
+      "loss": 0.0894,
+      "step": 1300
+    },
+    {
+      "epoch": 1.79,
+      "learning_rate": 2.2161422708618332e-06,
+      "loss": 0.1006,
+      "step": 1400
+    },
+    {
+      "epoch": 1.92,
+      "learning_rate": 8.481532147742819e-07,
+      "loss": 0.0767,
+      "step": 1500
+    },
+    {
+      "epoch": 2.0,
+      "eval_accuracy": 0.96628,
+      "eval_loss": 0.11105333268642426,
+      "eval_runtime": 750.9747,
+      "eval_samples_per_second": 33.29,
+      "eval_steps_per_second": 4.161,
+      "step": 1562
+    }
+  ],
+  "max_steps": 1562,
+  "num_train_epochs": 2,
+  "total_flos": 4.388029403227747e+16,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1d615fca9fe81200b28903fd357ad7f564201f1e497d29a22e86ac50ba458179
+size 4408