Model save

README.md

@@ -1,16 +1,12 @@
 ---
 license: llama3
-base_model: meta-llama/Meta-Llama-3-8B
+library_name: peft
 tags:
-- alignment-handbook
-- trl
-- sft
-- generated_from_trainer
 - trl
 - sft
+- alignment-handbook
 - generated_from_trainer
-datasets:
-- HuggingFaceH4/ultrachat_200k
+base_model: meta-llama/Meta-Llama-3-8B
 model-index:
 - name: llama5pEnt
   results: []
@@ -21,9 +17,7 @@ should probably proofread and complete it, then remove this comment. -->
 
 # llama5pEnt
 
-This model is a fine-tuned version of [meta-llama/Meta-Llama-3-8B](https://huggingface.co/meta-llama/Meta-Llama-3-8B) on the HuggingFaceH4/ultrachat_200k dataset.
-It achieves the following results on the evaluation set:
-- Loss: 1.0853
+This model is a fine-tuned version of [meta-llama/Meta-Llama-3-8B](https://huggingface.co/meta-llama/Meta-Llama-3-8B) on the None dataset.
 
 ## Model description
 
@@ -47,10 +41,10 @@ The following hyperparameters were used during training:
 - eval_batch_size: 4
 - seed: 42
 - distributed_type: multi-GPU
-- num_devices: 3
+- num_devices: 4
 - gradient_accumulation_steps: 8
-- total_train_batch_size: 192
-- total_eval_batch_size: 12
+- total_train_batch_size: 256
+- total_eval_batch_size: 16
 - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
 - lr_scheduler_type: cosine
 - lr_scheduler_warmup_ratio: 0.1
@@ -58,14 +52,12 @@ The following hyperparameters were used during training:
 
 ### Training results
 
-| Training Loss | Epoch  | Step | Validation Loss |
-|:-------------:|:------:|:----:|:---------------:|
-| 0.8639        | 0.9993 | 541  | 1.0853          |
 
 
 ### Framework versions
 
+- PEFT 0.11.1
 - Transformers 4.41.1
 - Pytorch 2.3.0+cu121
 - Datasets 2.19.1
-- Tokenizers 0.19.1
+- Tokenizers 0.19.1

adapter_config.json

@@ -0,0 +1,34 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "meta-llama/Meta-Llama-3-8B",
+  "bias": "none",
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 8,
+  "lora_dropout": 0.05,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 6,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "q_proj",
+    "o_proj",
+    "v_proj",
+    "k_proj",
+    "up_proj",
+    "down_proj",
+    "gate_proj"
+  ],
+  "task_type": "CAUSAL_LM",
+  "use_dora": false,
+  "use_rslora": false
+}

adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:41e632f4ebe506c85e162673d7f28cf041677a88d7d69aabaae11cd6bda63de0
+size 62973728

all_results.json

@@ -1,9 +1,9 @@
 {
-    "epoch": 0.9993073193257908,
-    "total_flos": 2.705944628841939e+18,
-    "train_loss": 0.860630649956229,
-    "train_runtime": 42250.2981,
+    "epoch": 0.9697529438928654,
+    "total_flos": 2.6262151886631076e+18,
+    "train_loss": 0.0,
+    "train_runtime": 0.0108,
     "train_samples": 103932,
-    "train_samples_per_second": 2.46,
-    "train_steps_per_second": 0.013
+    "train_samples_per_second": 9612825.336,
+    "train_steps_per_second": 37551.544
 }

runs/Jun05_21-11-52_COE-CS-sv003/events.out.tfevents.1717621973.COE-CS-sv003.73592.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b09c2cff4378d6e458ef4d1b0e3fd408de0b5654fa61c5cf7b09a083b4ffc7d1
+size 5414

tokenizer.json

@@ -1,11 +1,6 @@
 {
   "version": "1.0",
-  "truncation": {
-    "direction": "Right",
-    "max_length": 2048,
-    "strategy": "LongestFirst",
-    "stride": 0
-  },
+  "truncation": null,
   "padding": null,
   "added_tokens": [
     {

train_results.json

@@ -1,9 +1,9 @@
 {
-    "epoch": 0.9993073193257908,
-    "total_flos": 2.705944628841939e+18,
-    "train_loss": 0.860630649956229,
-    "train_runtime": 42250.2981,
+    "epoch": 0.9697529438928654,
+    "total_flos": 2.6262151886631076e+18,
+    "train_loss": 0.0,
+    "train_runtime": 0.0108,
     "train_samples": 103932,
-    "train_samples_per_second": 2.46,
-    "train_steps_per_second": 0.013
+    "train_samples_per_second": 9612825.336,
+    "train_steps_per_second": 37551.544
 }

trainer_state.json

@@ -1,9 +1,9 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.9993073193257908,
+  "epoch": 0.9697529438928654,
   "eval_steps": 500,
-  "global_step": 541,
+  "global_step": 525,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -751,46 +751,17 @@
       "step": 525
     },
     {
-      "epoch": 0.9789886862156546,
-      "grad_norm": 0.07811986654996872,
-      "learning_rate": 2.5269671064467313e-07,
-      "loss": 0.8473,
-      "step": 530
-    },
-    {
-      "epoch": 0.9882244285384437,
-      "grad_norm": 0.08186525851488113,
-      "learning_rate": 7.520474957699586e-08,
-      "loss": 0.8295,
-      "step": 535
-    },
-    {
-      "epoch": 0.997460170861233,
-      "grad_norm": 0.0831933468580246,
-      "learning_rate": 2.0892754394208346e-09,
-      "loss": 0.8639,
-      "step": 540
-    },
-    {
-      "epoch": 0.9993073193257908,
-      "eval_loss": 1.0852999687194824,
-      "eval_runtime": 177.1887,
-      "eval_samples_per_second": 6.518,
-      "eval_steps_per_second": 0.547,
-      "step": 541
-    },
-    {
-      "epoch": 0.9993073193257908,
-      "step": 541,
-      "total_flos": 2.705944628841939e+18,
-      "train_loss": 0.860630649956229,
-      "train_runtime": 42250.2981,
-      "train_samples_per_second": 2.46,
-      "train_steps_per_second": 0.013
+      "epoch": 0.9697529438928654,
+      "step": 525,
+      "total_flos": 2.6262151886631076e+18,
+      "train_loss": 0.0,
+      "train_runtime": 0.0108,
+      "train_samples_per_second": 9612825.336,
+      "train_steps_per_second": 37551.544
     }
   ],
   "logging_steps": 5,
-  "max_steps": 541,
+  "max_steps": 406,
   "num_input_tokens_seen": 0,
   "num_train_epochs": 1,
   "save_steps": 25,
@@ -806,7 +777,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 2.705944628841939e+18,
+  "total_flos": 2.6262151886631076e+18,
   "train_batch_size": 8,
   "trial_name": null,
   "trial_params": null

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e2ab12f53525aafe95fffa603a0d5583fdbabfc4b48c9c2446ab0ea6cb634f54
+oid sha256:77a3354f1d610f0c132b7cc50e46bd6980ccf70fd83feddaf7ac8dbfc6700277
 size 5240