Training in progress, step 200, checkpoint

.gitattributes

@@ -38,3 +38,8 @@ checkpoint-100/optimizer_0/__0_0.distcp filter=lfs diff=lfs merge=lfs -text
 checkpoint-100/optimizer_0/__1_0.distcp filter=lfs diff=lfs merge=lfs -text
 checkpoint-100/pytorch_model_fsdp_0/__0_0.distcp filter=lfs diff=lfs merge=lfs -text
 checkpoint-100/pytorch_model_fsdp_0/__1_0.distcp filter=lfs diff=lfs merge=lfs -text
+checkpoint-200/optimizer_0/.metadata filter=lfs diff=lfs merge=lfs -text
+checkpoint-200/optimizer_0/__0_0.distcp filter=lfs diff=lfs merge=lfs -text
+checkpoint-200/optimizer_0/__1_0.distcp filter=lfs diff=lfs merge=lfs -text
+checkpoint-200/pytorch_model_fsdp_0/__0_0.distcp filter=lfs diff=lfs merge=lfs -text
+checkpoint-200/pytorch_model_fsdp_0/__1_0.distcp filter=lfs diff=lfs merge=lfs -text

checkpoint-200/optimizer_0/.metadata

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a74ba8fcf2d857e573ced9e8ccd472ece612ef1ca47c4379e8bbc05bf43f4fa8
+size 2108254

checkpoint-200/optimizer_0/__0_0.distcp

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:674032aaae689e20caaff7e152cbd81a796a39b1720c9d8499e2e8bfe020d17c
+size 13256787644

checkpoint-200/optimizer_0/__1_0.distcp

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e7f0533ae806729f12554e1303ee02f6956c3b534ee929f5f3a4bf1db4434793
+size 13257964260

checkpoint-200/pytorch_model_fsdp_0/__0_0.distcp

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f99269a6b3feba7dcd3885f3d103cdec8bb7a514423ae942dfd7db8b418e8225
+size 6628321920

checkpoint-200/pytorch_model_fsdp_0/__1_0.distcp

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c5dd01741a4ccc7edbdfc39cfd832497ccf67fa73c32959b4325d940eb95fb46
+size 6628321920

checkpoint-200/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:985be3c100966af62f00b8631c27deac90e52e423a7e4a2c3255b59f4da8eee1
+oid sha256:41cb1fd0cf02312df7c9db483296344d5cea23a620661cf3977d37a50cab42f0
 size 14512

checkpoint-200/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f1e9b24a21ea7d3a361422caff7c5c58bb120577ea3bdc81d81a35c85fe0c4b8
+oid sha256:4db18a80bfe3ec5bfeabaed27eedc5b6daa844f4842bf328e9ce888aeb18adc5
 size 14512

checkpoint-200/trainer_state.json

@@ -1,7 +1,7 @@
 {
-  "best_metric": 0.6097185015678406,
-  "best_model_checkpoint": "./llama3/28-08-24-Weni-Pipeline_test_Experiment with SFT and Llama3 70b-2_max_steps-1362_batch_16_2024-08-28/checkpoint-200",
-  "epoch": 0.4404073768235618,
+  "best_metric": 0.5742923021316528,
+  "best_model_checkpoint": "./llama3/30-08-24-Weni-Pipeline_test_Experiment with SFT and Llama3 70b-2_max_steps-1362_batch_8_2024-08-30/checkpoint-100",
+  "epoch": 0.8800880088008801,
   "eval_steps": 100,
   "global_step": 200,
   "is_hyper_param_search": false,
@@ -9,186 +9,186 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "epoch": 0.02202036884117809,
-      "grad_norm": 0.6965782642364502,
+      "epoch": 0.04400440044004401,
+      "grad_norm": 0.5568628907203674,
       "learning_rate": 7.5e-05,
-      "loss": 2.1201,
+      "loss": 2.0875,
       "step": 10
     },
     {
-      "epoch": 0.04404073768235618,
-      "grad_norm": 0.7504029870033264,
+      "epoch": 0.08800880088008801,
+      "grad_norm": 0.2537558972835541,
       "learning_rate": 0.00015,
-      "loss": 0.9392,
+      "loss": 0.9378,
       "step": 20
     },
     {
-      "epoch": 0.06606110652353427,
-      "grad_norm": 0.7269854545593262,
+      "epoch": 0.132013201320132,
+      "grad_norm": 0.24558919668197632,
       "learning_rate": 0.000225,
-      "loss": 0.7958,
+      "loss": 0.7,
       "step": 30
     },
     {
-      "epoch": 0.08808147536471236,
-      "grad_norm": 0.15891791880130768,
+      "epoch": 0.17601760176017603,
+      "grad_norm": 0.13937097787857056,
       "learning_rate": 0.0003,
-      "loss": 0.7251,
+      "loss": 0.6298,
       "step": 40
     },
     {
-      "epoch": 0.11010184420589045,
-      "grad_norm": 0.14764881134033203,
+      "epoch": 0.22002200220022003,
+      "grad_norm": 0.1871194988489151,
       "learning_rate": 0.00029995764763563235,
-      "loss": 0.6941,
+      "loss": 0.6321,
       "step": 50
     },
     {
-      "epoch": 0.13212221304706853,
-      "grad_norm": 0.11882930248975754,
+      "epoch": 0.264026402640264,
+      "grad_norm": 0.14626263082027435,
       "learning_rate": 0.00029983061445883305,
-      "loss": 0.673,
+      "loss": 0.6403,
       "step": 60
     },
     {
-      "epoch": 0.15414258188824662,
-      "grad_norm": 0.15152081847190857,
+      "epoch": 0.30803080308030806,
+      "grad_norm": 0.12049665302038193,
       "learning_rate": 0.0002996189722050073,
-      "loss": 0.6428,
+      "loss": 0.5998,
       "step": 70
     },
     {
-      "epoch": 0.17616295072942473,
-      "grad_norm": 0.1619480848312378,
+      "epoch": 0.35203520352035206,
+      "grad_norm": 0.13617923855781555,
       "learning_rate": 0.0002993228403881531,
-      "loss": 0.6465,
+      "loss": 0.5942,
       "step": 80
     },
     {
-      "epoch": 0.1981833195706028,
-      "grad_norm": 0.15019242465496063,
+      "epoch": 0.39603960396039606,
+      "grad_norm": 0.1271793246269226,
       "learning_rate": 0.00029894238623337174,
-      "loss": 0.6308,
+      "loss": 0.5647,
       "step": 90
     },
     {
-      "epoch": 0.2202036884117809,
-      "grad_norm": 0.14553773403167725,
+      "epoch": 0.44004400440044006,
+      "grad_norm": 0.18757876753807068,
       "learning_rate": 0.00029847782458243663,
-      "loss": 0.6314,
+      "loss": 0.5619,
       "step": 100
     },
     {
-      "epoch": 0.2202036884117809,
+      "epoch": 0.44004400440044006,
       "eval_accuracy": 1.0,
       "eval_f1": 1.0,
       "eval_f1_macro": 1.0,
       "eval_f1_micro": 1.0,
-      "eval_loss": 0.6374139189720154,
+      "eval_loss": 0.5742923021316528,
       "eval_precision": 1.0,
       "eval_precision_macro": 1.0,
       "eval_precision_micro": 1.0,
       "eval_recall": 1.0,
       "eval_recall_macro": 1.0,
       "eval_recall_micro": 1.0,
-      "eval_runtime": 1957.1676,
-      "eval_samples_per_second": 0.413,
-      "eval_steps_per_second": 0.206,
+      "eval_runtime": 90.5857,
+      "eval_samples_per_second": 4.46,
+      "eval_steps_per_second": 1.115,
       "step": 100
     },
     {
-      "epoch": 0.24222405725295898,
-      "grad_norm": 0.20665155351161957,
+      "epoch": 0.48404840484048406,
+      "grad_norm": 0.14132679998874664,
       "learning_rate": 0.00029792941777247184,
-      "loss": 0.646,
+      "loss": 0.5584,
       "step": 110
     },
     {
-      "epoch": 0.26424442609413706,
-      "grad_norm": 0.18165886402130127,
+      "epoch": 0.528052805280528,
+      "grad_norm": 0.15474887192249298,
       "learning_rate": 0.0002972974754878111,
-      "loss": 0.6326,
+      "loss": 0.5752,
       "step": 120
     },
     {
-      "epoch": 0.28626479493531515,
-      "grad_norm": 0.13634872436523438,
+      "epoch": 0.5720572057205721,
+      "grad_norm": 0.13014496862888336,
       "learning_rate": 0.0002965823545851199,
-      "loss": 0.5785,
+      "loss": 0.5565,
       "step": 130
     },
     {
-      "epoch": 0.30828516377649323,
-      "grad_norm": 0.14753705263137817,
+      "epoch": 0.6160616061606161,
+      "grad_norm": 0.12456662207841873,
       "learning_rate": 0.00029578445889187865,
-      "loss": 0.639,
+      "loss": 0.5722,
       "step": 140
     },
     {
-      "epoch": 0.33030553261767137,
-      "grad_norm": 0.2674409747123718,
+      "epoch": 0.6600660066006601,
+      "grad_norm": 0.12824317812919617,
       "learning_rate": 0.00029490423897834234,
-      "loss": 0.626,
+      "loss": 0.523,
       "step": 150
     },
     {
-      "epoch": 0.35232590145884946,
-      "grad_norm": 0.16381941735744476,
+      "epoch": 0.7040704070407041,
+      "grad_norm": 0.14279119670391083,
       "learning_rate": 0.0002939421919031044,
-      "loss": 0.6009,
+      "loss": 0.5523,
       "step": 160
     },
     {
-      "epoch": 0.37434627030002754,
-      "grad_norm": 0.1673993617296219,
+      "epoch": 0.7480748074807481,
+      "grad_norm": 0.11781885474920273,
       "learning_rate": 0.00029289886093240847,
-      "loss": 0.6193,
+      "loss": 0.5291,
       "step": 170
     },
     {
-      "epoch": 0.3963666391412056,
-      "grad_norm": 0.16463495790958405,
+      "epoch": 0.7920792079207921,
+      "grad_norm": 0.1608349233865738,
       "learning_rate": 0.0002917748352333667,
-      "loss": 0.5894,
+      "loss": 0.5417,
       "step": 180
     },
     {
-      "epoch": 0.4183870079823837,
-      "grad_norm": 0.17818249762058258,
+      "epoch": 0.8360836083608361,
+      "grad_norm": 0.13777320086956024,
       "learning_rate": 0.0002905707495412589,
-      "loss": 0.6249,
+      "loss": 0.4967,
       "step": 190
     },
     {
-      "epoch": 0.4404073768235618,
-      "grad_norm": 0.14480963349342346,
+      "epoch": 0.8800880088008801,
+      "grad_norm": 0.21577192842960358,
       "learning_rate": 0.00028928728380109764,
-      "loss": 0.6223,
+      "loss": 0.6545,
       "step": 200
     },
     {
-      "epoch": 0.4404073768235618,
+      "epoch": 0.8800880088008801,
       "eval_accuracy": 1.0,
       "eval_f1": 1.0,
       "eval_f1_macro": 1.0,
       "eval_f1_micro": 1.0,
-      "eval_loss": 0.6097185015678406,
+      "eval_loss": 0.6772989630699158,
       "eval_precision": 1.0,
       "eval_precision_macro": 1.0,
       "eval_precision_micro": 1.0,
       "eval_recall": 1.0,
       "eval_recall_macro": 1.0,
       "eval_recall_micro": 1.0,
-      "eval_runtime": 1863.6968,
-      "eval_samples_per_second": 0.434,
-      "eval_steps_per_second": 0.217,
+      "eval_runtime": 90.2067,
+      "eval_samples_per_second": 4.479,
+      "eval_steps_per_second": 1.12,
       "step": 200
     }
   ],
   "logging_steps": 10,
   "max_steps": 1362,
   "num_input_tokens_seen": 0,
-  "num_train_epochs": 3,
+  "num_train_epochs": 6,
   "save_steps": 100,
   "stateful_callbacks": {
     "TrainerControl": {
@@ -202,7 +202,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 9495153475584.0,
+  "total_flos": 2.774286031954903e+17,
   "train_batch_size": 2,
   "trial_name": null,
   "trial_params": null