Training in progress epoch 0

README.md

@@ -1,6 +1,5 @@
 ---
-license: apache-2.0
-base_model: distilbert-base-multilingual-cased
+base_model: monologg/koelectra-small-finetuned-sentiment
 tags:
 - generated_from_keras_callback
 model-index:
@@ -13,10 +12,11 @@ probably proofread and complete it, then remove this comment. -->
 
 # CYONG/v1
 
-This model is a fine-tuned version of [distilbert-base-multilingual-cased](https://huggingface.co/distilbert-base-multilingual-cased) on an unknown dataset.
+This model is a fine-tuned version of [monologg/koelectra-small-finetuned-sentiment](https://huggingface.co/monologg/koelectra-small-finetuned-sentiment) on an unknown dataset.
 It achieves the following results on the evaluation set:
-- Train Loss: 1.3818
-- Validation Loss: 0.8440
+- Train Loss: 3.0778
+- Validation Loss: 3.0413
+- Train Accuracy: 0.5828
 - Epoch: 0
 
 ## Model description
@@ -36,14 +36,14 @@ More information needed
 ### Training hyperparameters
 
 The following hyperparameters were used during training:
-- optimizer: {'name': 'Adam', 'weight_decay': None, 'clipnorm': None, 'global_clipnorm': None, 'clipvalue': None, 'use_ema': False, 'ema_momentum': 0.99, 'ema_overwrite_frequency': None, 'jit_compile': True, 'is_legacy_optimizer': False, 'learning_rate': {'module': 'keras.optimizers.schedules', 'class_name': 'PolynomialDecay', 'config': {'initial_learning_rate': 2e-05, 'decay_steps': 1510100, 'end_learning_rate': 0.0, 'power': 1.0, 'cycle': False, 'name': None}, 'registered_name': None}, 'beta_1': 0.9, 'beta_2': 0.999, 'epsilon': 1e-08, 'amsgrad': False}
+- optimizer: {'name': 'Adam', 'weight_decay': None, 'clipnorm': None, 'global_clipnorm': None, 'clipvalue': None, 'use_ema': False, 'ema_momentum': 0.99, 'ema_overwrite_frequency': None, 'jit_compile': True, 'is_legacy_optimizer': False, 'learning_rate': {'module': 'keras.optimizers.schedules', 'class_name': 'PolynomialDecay', 'config': {'initial_learning_rate': 2e-05, 'decay_steps': 95, 'end_learning_rate': 0.0, 'power': 1.0, 'cycle': False, 'name': None}, 'registered_name': None}, 'beta_1': 0.9, 'beta_2': 0.999, 'epsilon': 1e-08, 'amsgrad': False}
 - training_precision: float32
 
 ### Training results
 
-| Train Loss | Validation Loss | Epoch |
-|:----------:|:---------------:|:-----:|
-| 1.3818     | 0.8440          | 0     |
+| Train Loss | Validation Loss | Train Accuracy | Epoch |
+|:----------:|:---------------:|:--------------:|:-----:|
+| 3.0778     | 3.0413          | 0.5828         | 0     |
 
 
 ### Framework versions

config.json

@@ -1,24 +1,79 @@
 {
-  "_name_or_path": "distilbert-base-multilingual-cased",
-  "activation": "gelu",
+  "_name_or_path": "monologg/koelectra-small-finetuned-sentiment",
+  "_num_labels": 2,
   "architectures": [
-    "DistilBertForQuestionAnswering"
+    "ElectraForSequenceClassification"
   ],
-  "attention_dropout": 0.1,
-  "dim": 768,
-  "dropout": 0.1,
-  "hidden_dim": 3072,
+  "attention_probs_dropout_prob": 0.1,
+  "classifier_dropout": null,
+  "embedding_size": 128,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 256,
+  "id2label": {
+    "0": "quota",
+    "1": "\uae30\ubd80 \uc694\uccad",
+    "2": "\uac8c\uc2dc\uae00",
+    "3": "\ub178\ud2b8",
+    "4": "\uc774\ubbf8\uc9c0",
+    "5": "\uc804\uccb4",
+    "6": "\ucd2c\uc601",
+    "7": "\ubb38\uc11c\uc778\uc2dd",
+    "8": "\ube5b \ubc1d\uae30",
+    "9": "\ub354\ubcf4\uae30",
+    "10": "\ubb38\uc790\uc2a4\uce94",
+    "11": "AI\ubaa8\ub4dc",
+    "12": "\uc0c9\uc0c1\uc778\uc2dd",
+    "13": "\ud604\uc7ac\ubaa8\ub4dc",
+    "14": "\uc774\ubbf8\uc9c0\ubb18\uc0ac",
+    "15": "\ubb3c\uac74\ucc3e\uae30",
+    "16": "\uc5bc\uad74\uc778\uc2dd",
+    "17": "\ub9ac\uc5bc\uc544\uc774\uc988",
+    "18": "\uc637\uc778\uc2dd",
+    "19": "\uc9c0\ud3d0\uc778\uc2dd",
+    "20": "\ub3cb\ubcf4\uae30",
+    "21": "\ubb38\uc790\uc778\uc2dd"
+  },
   "initializer_range": 0.02,
+  "intermediate_size": 1024,
+  "label2id": {
+    "AI\ubaa8\ub4dc": 11,
+    "quota": 0,
+    "\uac8c\uc2dc\uae00": 2,
+    "\uae30\ubd80 \uc694\uccad": 1,
+    "\ub178\ud2b8": 3,
+    "\ub354\ubcf4\uae30": 9,
+    "\ub3cb\ubcf4\uae30": 20,
+    "\ub9ac\uc5bc\uc544\uc774\uc988": 17,
+    "\ubb38\uc11c\uc778\uc2dd": 7,
+    "\ubb38\uc790\uc2a4\uce94": 10,
+    "\ubb38\uc790\uc778\uc2dd": 21,
+    "\ubb3c\uac74\ucc3e\uae30": 15,
+    "\ube5b \ubc1d\uae30": 8,
+    "\uc0c9\uc0c1\uc778\uc2dd": 12,
+    "\uc5bc\uad74\uc778\uc2dd": 16,
+    "\uc637\uc778\uc2dd": 18,
+    "\uc774\ubbf8\uc9c0": 4,
+    "\uc774\ubbf8\uc9c0\ubb18\uc0ac": 14,
+    "\uc804\uccb4": 5,
+    "\uc9c0\ud3d0\uc778\uc2dd": 19,
+    "\ucd2c\uc601": 6,
+    "\ud604\uc7ac\ubaa8\ub4dc": 13
+  },
+  "layer_norm_eps": 1e-12,
   "max_position_embeddings": 512,
-  "model_type": "distilbert",
-  "n_heads": 12,
-  "n_layers": 6,
+  "model_type": "electra",
+  "num_attention_heads": 4,
+  "num_hidden_layers": 12,
   "output_past": true,
   "pad_token_id": 0,
-  "qa_dropout": 0.1,
-  "seq_classif_dropout": 0.2,
-  "sinusoidal_pos_embds": false,
-  "tie_weights_": true,
+  "position_embedding_type": "absolute",
+  "summary_activation": "gelu",
+  "summary_last_dropout": 0.1,
+  "summary_type": "first",
+  "summary_use_proj": true,
   "transformers_version": "4.31.0",
-  "vocab_size": 119547
+  "type_vocab_size": 2,
+  "use_cache": true,
+  "vocab_size": 32200
 }

tf_model.h5

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:09d8225d6a34531f6da389f7cf1f2d41b46bf3c68481d27c48f030c780c247f7
-size 539068456
+oid sha256:1092d6e85ccdab854d2f6ba0c2be3b92f1674eef878ed2c20d131d3bce337161
+size 55349584

tokenizer_config.json

@@ -1,13 +1,16 @@
 {
   "clean_up_tokenization_spaces": true,
   "cls_token": "[CLS]",
+  "do_basic_tokenize": true,
   "do_lower_case": false,
   "mask_token": "[MASK]",
+  "max_len": 512,
   "model_max_length": 512,
+  "never_split": null,
   "pad_token": "[PAD]",
   "sep_token": "[SEP]",
   "strip_accents": null,
   "tokenize_chinese_chars": true,
-  "tokenizer_class": "DistilBertTokenizer",
+  "tokenizer_class": "ElectraTokenizer",
   "unk_token": "[UNK]"
 }