Upload 12 files

README.md

@@ -0,0 +1,14 @@
+`google/electra-large-discriminator` finetuned for regression on imdb dataset ratings for 3 epoches.
+
+Large examples tokenized with head and tail parts of a review, as described in [How to Fine-Tune BERT for Text Classification?](https://arxiv.org/abs/1905.05583)
+
+```python
+def preprocess_function(example):
+    tokens = tokenizer(example["text"], truncation=False)
+    if len(tokens['input_ids']) > 512:
+        tokens['input_ids'] = tokens['input_ids'][:129] + \
+            [102] + tokens['input_ids'][-382:]
+        tokens['token_type_ids'] = [0]*512
+        tokens['attention_mask'] = [1]*512
+    return tokens
+```

config.json

@@ -0,0 +1,37 @@
+{
+  "_name_or_path": "google/electra-large-discriminator",
+  "architectures": [
+    "ElectraForSequenceClassification"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "classifier_dropout": null,
+  "embedding_size": 1024,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 1024,
+  "id2label": {
+    "0": "LABEL_0"
+  },
+  "initializer_range": 0.02,
+  "intermediate_size": 4096,
+  "label2id": {
+    "LABEL_0": 0
+  },
+  "layer_norm_eps": 1e-12,
+  "max_position_embeddings": 512,
+  "model_type": "electra",
+  "num_attention_heads": 16,
+  "num_hidden_layers": 24,
+  "pad_token_id": 0,
+  "position_embedding_type": "absolute",
+  "problem_type": "regression",
+  "summary_activation": "gelu",
+  "summary_last_dropout": 0.1,
+  "summary_type": "first",
+  "summary_use_proj": true,
+  "torch_dtype": "float32",
+  "transformers_version": "4.31.0",
+  "type_vocab_size": 2,
+  "use_cache": true,
+  "vocab_size": 30522
+}

optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c89364d8b1c09caedd45a8e54f159a9c620b5b98691d69c90010bd614e0b78d7
+size 2681374650

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f33ac37eebb186ec57c905a3d78b2be0f113dc4e7ee5b398772b8743f40d224b
+size 1340707694

rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fbcedcf49f3f9589fbd7ec19a42015e921e27848b919d7b5fe8642b3b0d6897d
+size 14244

scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a64b1d144af0255439fc1c8acba2eb9e37854f720c7eb58713e093777f8249b3
+size 1064

special_tokens_map.json

@@ -0,0 +1,7 @@
+{
+  "cls_token": "[CLS]",
+  "mask_token": "[MASK]",
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "unk_token": "[UNK]"
+}

tokenizer_config.json

@@ -0,0 +1,13 @@
+{
+  "clean_up_tokenization_spaces": true,
+  "cls_token": "[CLS]",
+  "do_lower_case": true,
+  "mask_token": "[MASK]",
+  "model_max_length": 512,
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "strip_accents": null,
+  "tokenize_chinese_chars": true,
+  "tokenizer_class": "ElectraTokenizer",
+  "unk_token": "[UNK]"
+}

trainer_state.json

@@ -0,0 +1,322 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 2.99904,
+  "global_step": 2343,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.06,
+      "learning_rate": 1e-05,
+      "loss": 36.9779,
+      "step": 50
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 2e-05,
+      "loss": 16.6502,
+      "step": 100
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 1.9554168524297815e-05,
+      "loss": 13.7007,
+      "step": 150
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 1.9108337048595635e-05,
+      "loss": 6.0772,
+      "step": 200
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 1.8662505572893448e-05,
+      "loss": 4.2448,
+      "step": 250
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 1.8216674097191264e-05,
+      "loss": 5.0142,
+      "step": 300
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 1.777084262148908e-05,
+      "loss": 3.3927,
+      "step": 350
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 1.7325011145786894e-05,
+      "loss": 3.4273,
+      "step": 400
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 1.687917967008471e-05,
+      "loss": 2.906,
+      "step": 450
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 1.6433348194382527e-05,
+      "loss": 2.6533,
+      "step": 500
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 1.598751671868034e-05,
+      "loss": 2.5707,
+      "step": 550
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 1.5541685242978156e-05,
+      "loss": 2.5857,
+      "step": 600
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 1.5095853767275971e-05,
+      "loss": 2.5658,
+      "step": 650
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.4650022291573786e-05,
+      "loss": 2.7225,
+      "step": 700
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 1.42041908158716e-05,
+      "loss": 2.0151,
+      "step": 750
+    },
+    {
+      "epoch": 1.0,
+      "eval_accuracy": 0.88208,
+      "eval_loss": 3.9117867946624756,
+      "eval_rmse": 4.946967789532184,
+      "eval_runtime": 621.0961,
+      "eval_samples_per_second": 40.251,
+      "eval_steps_per_second": 2.517,
+      "step": 781
+    },
+    {
+      "epoch": 1.02,
+      "learning_rate": 1.3758359340169416e-05,
+      "loss": 2.4671,
+      "step": 800
+    },
+    {
+      "epoch": 1.09,
+      "learning_rate": 1.3312527864467232e-05,
+      "loss": 1.8725,
+      "step": 850
+    },
+    {
+      "epoch": 1.15,
+      "learning_rate": 1.2866696388765047e-05,
+      "loss": 1.9538,
+      "step": 900
+    },
+    {
+      "epoch": 1.22,
+      "learning_rate": 1.2420864913062862e-05,
+      "loss": 1.7274,
+      "step": 950
+    },
+    {
+      "epoch": 1.28,
+      "learning_rate": 1.197503343736068e-05,
+      "loss": 1.9257,
+      "step": 1000
+    },
+    {
+      "epoch": 1.34,
+      "learning_rate": 1.1529201961658493e-05,
+      "loss": 1.8435,
+      "step": 1050
+    },
+    {
+      "epoch": 1.41,
+      "learning_rate": 1.1083370485956308e-05,
+      "loss": 1.6858,
+      "step": 1100
+    },
+    {
+      "epoch": 1.47,
+      "learning_rate": 1.0637539010254126e-05,
+      "loss": 1.9954,
+      "step": 1150
+    },
+    {
+      "epoch": 1.54,
+      "learning_rate": 1.019170753455194e-05,
+      "loss": 1.7033,
+      "step": 1200
+    },
+    {
+      "epoch": 1.6,
+      "learning_rate": 9.745876058849756e-06,
+      "loss": 1.4969,
+      "step": 1250
+    },
+    {
+      "epoch": 1.66,
+      "learning_rate": 9.30004458314757e-06,
+      "loss": 1.5146,
+      "step": 1300
+    },
+    {
+      "epoch": 1.73,
+      "learning_rate": 8.854213107445387e-06,
+      "loss": 2.0415,
+      "step": 1350
+    },
+    {
+      "epoch": 1.79,
+      "learning_rate": 8.408381631743202e-06,
+      "loss": 2.0009,
+      "step": 1400
+    },
+    {
+      "epoch": 1.86,
+      "learning_rate": 7.962550156041017e-06,
+      "loss": 1.7283,
+      "step": 1450
+    },
+    {
+      "epoch": 1.92,
+      "learning_rate": 7.516718680338832e-06,
+      "loss": 1.7819,
+      "step": 1500
+    },
+    {
+      "epoch": 1.98,
+      "learning_rate": 7.070887204636649e-06,
+      "loss": 1.5547,
+      "step": 1550
+    },
+    {
+      "epoch": 2.0,
+      "eval_accuracy": 0.95872,
+      "eval_loss": 1.6247199773788452,
+      "eval_rmse": 4.865195563248444,
+      "eval_runtime": 619.8743,
+      "eval_samples_per_second": 40.331,
+      "eval_steps_per_second": 2.521,
+      "step": 1562
+    },
+    {
+      "epoch": 2.05,
+      "learning_rate": 6.625055728934463e-06,
+      "loss": 1.4602,
+      "step": 1600
+    },
+    {
+      "epoch": 2.11,
+      "learning_rate": 6.179224253232279e-06,
+      "loss": 1.2891,
+      "step": 1650
+    },
+    {
+      "epoch": 2.18,
+      "learning_rate": 5.733392777530095e-06,
+      "loss": 1.2627,
+      "step": 1700
+    },
+    {
+      "epoch": 2.24,
+      "learning_rate": 5.28756130182791e-06,
+      "loss": 1.4664,
+      "step": 1750
+    },
+    {
+      "epoch": 2.3,
+      "learning_rate": 4.841729826125725e-06,
+      "loss": 1.2305,
+      "step": 1800
+    },
+    {
+      "epoch": 2.37,
+      "learning_rate": 4.39589835042354e-06,
+      "loss": 1.2872,
+      "step": 1850
+    },
+    {
+      "epoch": 2.43,
+      "learning_rate": 3.950066874721356e-06,
+      "loss": 1.2644,
+      "step": 1900
+    },
+    {
+      "epoch": 2.5,
+      "learning_rate": 3.5042353990191713e-06,
+      "loss": 1.3754,
+      "step": 1950
+    },
+    {
+      "epoch": 2.56,
+      "learning_rate": 3.0584039233169866e-06,
+      "loss": 1.1153,
+      "step": 2000
+    },
+    {
+      "epoch": 2.62,
+      "learning_rate": 2.612572447614802e-06,
+      "loss": 1.3498,
+      "step": 2050
+    },
+    {
+      "epoch": 2.69,
+      "learning_rate": 2.1667409719126175e-06,
+      "loss": 1.2545,
+      "step": 2100
+    },
+    {
+      "epoch": 2.75,
+      "learning_rate": 1.7209094962104325e-06,
+      "loss": 1.0703,
+      "step": 2150
+    },
+    {
+      "epoch": 2.82,
+      "learning_rate": 1.2750780205082481e-06,
+      "loss": 1.1519,
+      "step": 2200
+    },
+    {
+      "epoch": 2.88,
+      "learning_rate": 8.292465448060634e-07,
+      "loss": 1.0694,
+      "step": 2250
+    },
+    {
+      "epoch": 2.94,
+      "learning_rate": 3.834150691038788e-07,
+      "loss": 0.9306,
+      "step": 2300
+    },
+    {
+      "epoch": 3.0,
+      "eval_accuracy": 0.96212,
+      "eval_loss": 1.7369074821472168,
+      "eval_rmse": 5.055060677999024,
+      "eval_runtime": 635.5792,
+      "eval_samples_per_second": 39.334,
+      "eval_steps_per_second": 2.459,
+      "step": 2343
+    }
+  ],
+  "max_steps": 2343,
+  "num_train_epochs": 3,
+  "total_flos": 6.576905725215466e+16,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8c849546b7d1700f2171f51a0421215a781c89357d4c8c8797fb3aef84f224b2
+size 4408