Training in progress, step 1413

config.json

@@ -12,36 +12,38 @@
     "model_type": "blip_2_qformer"
   },
   "text_config": {
-    "_name_or_path": "beomi/kykim-gpt3-kor-small_based_on_gpt2",
+    "_name_or_path": "heegyu/kogpt-j-350m",
     "activation_function": "gelu_new",
     "architectures": [
-      "GPT2LMHeadModel"
+      "GPTJForCausalLM"
     ],
     "attn_pdrop": 0.1,
-    "bos_token_id": 3,
+    "bos_token_id": 1,
     "embd_pdrop": 0.1,
-    "eos_token_id": 3,
+    "eos_token_id": 2,
     "gradient_checkpointing": false,
     "initializer_range": 0.02,
     "layer_norm_epsilon": 1e-05,
-    "model_type": "gpt2",
-    "n_ctx": 2048,
-    "n_embd": 768,
-    "n_head": 12,
+    "model_type": "gptj",
+    "n_embd": 1024,
+    "n_head": 16,
     "n_inner": null,
-    "n_layer": 12,
-    "n_positions": 2048,
-    "pad_token_id": null,
-    "reorder_and_upcast_attn": false,
-    "resid_pdrop": 0.1,
-    "scale_attn_by_inverse_layer_idx": false,
+    "n_layer": 20,
+    "n_positions": 1024,
+    "pad_token_id": 0,
+    "resid_pdrop": 0.0,
+    "rotary_dim": 32,
     "scale_attn_weights": true,
     "summary_activation": null,
     "summary_first_dropout": 0.1,
     "summary_proj_to_labels": true,
     "summary_type": "cls_index",
     "summary_use_proj": true,
-    "vocab_size": 42000
+    "tie_word_embeddings": false,
+    "tokenizer_class": "GPT2Tokenizer",
+    "torch_dtype": "float32",
+    "unk_token_id": 5,
+    "vocab_size": 51200
   },
   "torch_dtype": "float32",
   "transformers_version": "4.36.2",

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6339a21b34644c86ad3b77dce16353de154467dcf78d87de112129ea019f77ba
+oid sha256:f98980e0f1387d5822a266be8662e657ae89fe4521ca7a72e28e1b15b8f2eabf
 size 4664