Training in progress, epoch 1, checkpoint

last-checkpoint/config.json

@@ -1,32 +1,31 @@
 {
-  "_name_or_path": "ccore/getcode-350m",
+  "_name_or_path": "facebook/opt-350m",
+  "_remove_final_layer_norm": false,
+  "activation_dropout": 0.0,
+  "activation_function": "relu",
   "architectures": [
-    "LlamaForCausalLM"
+    "OPTForCausalLM"
   ],
-  "attention_bias": false,
   "attention_dropout": 0.0,
-  "bos_token_id": 0,
-  "eos_token_id": 0,
-  "head_dim": 64,
-  "hidden_act": "silu",
-  "hidden_size": 960,
-  "initializer_range": 0.02,
-  "intermediate_size": 2560,
-  "is_llama_config": true,
-  "max_position_embeddings": 8192,
-  "mlp_bias": false,
-  "model_type": "llama",
-  "num_attention_heads": 15,
-  "num_hidden_layers": 32,
-  "num_key_value_heads": 5,
-  "pretraining_tp": 1,
-  "rms_norm_eps": 1e-05,
-  "rope_interleaved": false,
-  "rope_scaling": null,
-  "rope_theta": 100000,
-  "tie_word_embeddings": true,
+  "bos_token_id": 2,
+  "do_layer_norm_before": false,
+  "dropout": 0.1,
+  "enable_bias": true,
+  "eos_token_id": 2,
+  "ffn_dim": 4096,
+  "hidden_size": 1024,
+  "init_std": 0.02,
+  "layer_norm_elementwise_affine": true,
+  "layerdrop": 0.0,
+  "max_position_embeddings": 2048,
+  "model_type": "opt",
+  "num_attention_heads": 16,
+  "num_hidden_layers": 24,
+  "pad_token_id": 1,
+  "prefix": "</s>",
   "torch_dtype": "float32",
   "transformers_version": "4.47.0",
   "use_cache": true,
-  "vocab_size": 49152
+  "vocab_size": 50272,
+  "word_embed_proj_dim": 512
 }

last-checkpoint/generation_config.json

@@ -1,6 +1,7 @@
 {
   "_from_model_config": true,
-  "bos_token_id": 0,
-  "eos_token_id": 0,
+  "bos_token_id": 2,
+  "eos_token_id": 2,
+  "pad_token_id": 1,
   "transformers_version": "4.47.0"
 }

last-checkpoint/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4ab237cb7a4a12b5913039477bbfb178aac5c8809b3264af22d9311f26c092d4
-size 1447317080
+oid sha256:1bf1d2a952d87403172e62da7597cdc8a96002c708f0a5e79d7de04f4584bf8e
+size 1324830880

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c7587ea383570bccad6d8c4d94834efb8793ba6d29c3ce414358d378cac00fe4
-size 2894813242
+oid sha256:0b2edf4f04bf9b4035f407e950f30565e1809557f5a012ef19f0215deee1a206
+size 2649896030

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d245e05e72192c132e0f2edb6fdcae0c578c890f0fe912f17ec7b0bba2d38cc3
+oid sha256:df28232a1de30d1b227c0efb54de7a0d430617f62ba514c9422d3e8b85d3ced8
 size 14244

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0c785df7642646aef2a39d5240ed589e73b085e3b34051de846243fbbdb4deb6
+oid sha256:2dbf8f944ea194273642caaf2a78a4ac235bd542ada6976e2803f1735daa8e77
 size 1064

last-checkpoint/trainer_state.json

@@ -1,112 +1,24 @@
 {
-  "best_metric": 0.344450980424881,
-  "best_model_checkpoint": "./opt_trained2/checkpoint-803",
-  "epoch": 5.0,
+  "best_metric": 2.4022321701049805,
+  "best_model_checkpoint": "./opt_trained1/checkpoint-268",
+  "epoch": 1.0,
   "eval_steps": 500,
-  "global_step": 4015,
+  "global_step": 268,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
-    {
-      "epoch": 0.6226650062266501,
-      "grad_norm": 1.3505125045776367,
-      "learning_rate": 0.000387546699875467,
-      "loss": 2.7763,
-      "step": 500
-    },
     {
       "epoch": 1.0,
-      "eval_loss": 0.344450980424881,
-      "eval_runtime": 273.127,
-      "eval_samples_per_second": 10.457,
-      "eval_steps_per_second": 2.614,
-      "step": 803
-    },
-    {
-      "epoch": 1.2453300124533002,
-      "grad_norm": 1.1673122644424438,
-      "learning_rate": 0.000375093399750934,
-      "loss": 2.6965,
-      "step": 1000
-    },
-    {
-      "epoch": 1.86799501867995,
-      "grad_norm": 0.7170485258102417,
-      "learning_rate": 0.000362640099626401,
-      "loss": 2.5469,
-      "step": 1500
-    },
-    {
-      "epoch": 2.0,
-      "eval_loss": 0.3459263741970062,
-      "eval_runtime": 272.8846,
-      "eval_samples_per_second": 10.466,
-      "eval_steps_per_second": 2.616,
-      "step": 1606
-    },
-    {
-      "epoch": 2.4906600249066004,
-      "grad_norm": 1.186563491821289,
-      "learning_rate": 0.000350186799501868,
-      "loss": 2.2782,
-      "step": 2000
-    },
-    {
-      "epoch": 3.0,
-      "eval_loss": 0.366046667098999,
-      "eval_runtime": 272.9944,
-      "eval_samples_per_second": 10.462,
-      "eval_steps_per_second": 2.615,
-      "step": 2409
-    },
-    {
-      "epoch": 3.1133250311332503,
-      "grad_norm": 1.2075772285461426,
-      "learning_rate": 0.000337733499377335,
-      "loss": 2.1429,
-      "step": 2500
-    },
-    {
-      "epoch": 3.7359900373599,
-      "grad_norm": 1.2370275259017944,
-      "learning_rate": 0.00032528019925280203,
-      "loss": 1.7732,
-      "step": 3000
-    },
-    {
-      "epoch": 4.0,
-      "eval_loss": 0.41189202666282654,
-      "eval_runtime": 272.6721,
-      "eval_samples_per_second": 10.474,
-      "eval_steps_per_second": 2.619,
-      "step": 3212
-    },
-    {
-      "epoch": 4.35865504358655,
-      "grad_norm": 1.3971993923187256,
-      "learning_rate": 0.000312826899128269,
-      "loss": 1.5169,
-      "step": 3500
-    },
-    {
-      "epoch": 4.981320049813201,
-      "grad_norm": 1.8089447021484375,
-      "learning_rate": 0.000300373599003736,
-      "loss": 1.3553,
-      "step": 4000
-    },
-    {
-      "epoch": 5.0,
-      "eval_loss": 0.5129567384719849,
-      "eval_runtime": 272.4812,
-      "eval_samples_per_second": 10.481,
-      "eval_steps_per_second": 2.62,
-      "step": 4015
+      "eval_loss": 2.4022321701049805,
+      "eval_runtime": 207.1452,
+      "eval_samples_per_second": 13.787,
+      "eval_steps_per_second": 1.723,
+      "step": 268
     }
   ],
   "logging_steps": 500,
-  "max_steps": 16060,
+  "max_steps": 5340,
   "num_input_tokens_seen": 0,
   "num_train_epochs": 20,
   "save_steps": 500,
@@ -122,8 +34,8 @@
       "attributes": {}
     }
   },
-  "total_flos": 1.564360708284e+17,
-  "train_batch_size": 4,
+  "total_flos": 1.809948514295808e+16,
+  "train_batch_size": 12,
   "trial_name": null,
   "trial_params": null
 }

last-checkpoint/training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:83ebdc8e79af6a5c9fa4062f9f21aa548e32853f463a515333ea4cf561b7abfc
+oid sha256:f5db2222136e05949276be8a807dc409239b47ab761a0c7e53aa25bc62897fc5
 size 5368