Update config.json

config.json

@@ -1,49 +1,33 @@
 {
-    "model_type": "custom_transformer",
-    "architectures": ["CustomTransformer"],
-    "vocab_size": 50257,
-    "hidden_size": 768,
-    "num_hidden_layers": 8,
-    "num_attention_heads": 8,
-    "intermediate_size": 3072,
-    "hidden_act": "gelu",
-    "hidden_dropout_prob": 0.1,
-    "max_position_embeddings": 512,
-    "initializer_range": 0.02,
-    "layer_norm_eps": 1e-12,
-    "pad_token_id": 50256,
-    "bos_token_id": 50256,
-    "eos_token_id": 50256,
-    "transformers_version": "4.28.1",
-    "rl_params": {
-        "clip_epsilon": 0.2,
-        "value_coef": 0.5,
-        "entropy_coef": 0.01,
-        "ppo_epochs": 4,
-        "gamma": 0.99,
-        "lam": 0.95
-    },
-    "training_params": {
-        "learning_rate": 5e-5,
-        "train_batch_size": 32,
-        "warmup_steps": 500,
-        "weight_decay": 0.01,
-        "max_grad_norm": 1.0,
-        "lr_scheduler_type": "cosine"
-    },
-    "dataset_params": {
-        "name": "roneneldan/TinyStories",
-        "max_seq_length": 512,
-        "text_column": "text",
-        "subset_percentage": 10
-    },
-    "tokenizer_params": {
-        "pretrained_tokenizer": "gpt2",
-        "padding_side": "right"
-    },
-    "reward_model": {
-        "model_name": "distilbert-base-uncased",
-        "num_labels": 1,
-        "score_threshold": 0.5
-    }
-}
+  "activation_function": "gelu_new",
+  "architectures": [
+    "GPT2LMHeadModel"
+  ],
+  "attn_pdrop": 0.1,
+  "bos_token_id": 50256,
+  "embd_pdrop": 0.1,
+  "eos_token_id": 50256,
+  "initializer_range": 0.02,
+  "layer_norm_epsilon": 1e-05,
+  "model_type": "gpt2",
+  "n_ctx": 512,
+  "n_embd": 512,
+  "n_head": 8,
+  "n_inner": null,
+  "n_layer": 8,
+  "n_positions": 768,
+  "pad_token_id": 50256,
+  "reorder_and_upcast_attn": false,
+  "resid_pdrop": 0.1,
+  "scale_attn_by_inverse_layer_idx": false,
+  "scale_attn_weights": true,
+  "summary_activation": null,
+  "summary_first_dropout": 0.1,
+  "summary_proj_to_labels": true,
+  "summary_type": "cls_index",
+  "summary_use_proj": true,
+  "torch_dtype": "float32",
+  "transformers_version": "4.47.1",
+  "use_cache": true,
+  "vocab_size": 50257
+}