Model save

README.md

@@ -0,0 +1,69 @@
+---
+library_name: peft
+license: llama2
+base_model: meta-llama/Llama-2-13b-hf
+tags:
+- trl
+- sft
+- generated_from_trainer
+datasets:
+- generator
+model-index:
+- name: llama2-13b-lora-coding-11-v1
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# llama2-13b-lora-coding-11-v1
+
+This model is a fine-tuned version of [meta-llama/Llama-2-13b-hf](https://huggingface.co/meta-llama/Llama-2-13b-hf) on the generator dataset.
+It achieves the following results on the evaluation set:
+- Loss: 1.4634
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 0.0002
+- train_batch_size: 32
+- eval_batch_size: 32
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 8
+- gradient_accumulation_steps: 2
+- total_train_batch_size: 512
+- total_eval_batch_size: 256
+- optimizer: Use adamw_torch with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
+- lr_scheduler_type: cosine
+- lr_scheduler_warmup_ratio: 0.1
+- num_epochs: 1
+
+### Training results
+
+| Training Loss | Epoch  | Step | Validation Loss |
+|:-------------:|:------:|:----:|:---------------:|
+| 1.0723        | 0.9938 | 80   | 1.4634          |
+
+
+### Framework versions
+
+- PEFT 0.13.2
+- Transformers 4.46.2
+- Pytorch 2.5.1+cu124
+- Datasets 3.1.0
+- Tokenizers 0.20.3

all_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 0.9937888198757764,
+    "total_flos": 1.617984821877801e+18,
+    "train_loss": 1.690801477432251,
+    "train_runtime": 1102.9963,
+    "train_samples": 116368,
+    "train_samples_per_second": 37.273,
+    "train_steps_per_second": 0.073
+}

runs/Nov22_15-17-52_metallic-vm-falls-fin-02/events.out.tfevents.1732289044.metallic-vm-falls-fin-02.66981.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f1ad0b87b487270af306ee33be4efccff76e013c1bfb4e824a75bddfe0c5e4c7
-size 9124
+oid sha256:dd6cea688ba12a14e87039bbaa2176ed5aeac3904cb5078230d61a3a639878d6
+size 9738

train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 0.9937888198757764,
+    "total_flos": 1.617984821877801e+18,
+    "train_loss": 1.690801477432251,
+    "train_runtime": 1102.9963,
+    "train_samples": 116368,
+    "train_samples_per_second": 37.273,
+    "train_steps_per_second": 0.073
+}

trainer_state.json

@@ -0,0 +1,169 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.9937888198757764,
+  "eval_steps": 500,
+  "global_step": 80,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.012422360248447204,
+      "grad_norm": 2.1266329288482666,
+      "learning_rate": 2.5e-05,
+      "loss": 4.2081,
+      "step": 1
+    },
+    {
+      "epoch": 0.062111801242236024,
+      "grad_norm": 2.5932071208953857,
+      "learning_rate": 0.000125,
+      "loss": 4.1269,
+      "step": 5
+    },
+    {
+      "epoch": 0.12422360248447205,
+      "grad_norm": 3.8309450149536133,
+      "learning_rate": 0.00019961946980917456,
+      "loss": 3.5809,
+      "step": 10
+    },
+    {
+      "epoch": 0.18633540372670807,
+      "grad_norm": 3.414670944213867,
+      "learning_rate": 0.0001953716950748227,
+      "loss": 2.7605,
+      "step": 15
+    },
+    {
+      "epoch": 0.2484472049689441,
+      "grad_norm": 2.252514600753784,
+      "learning_rate": 0.00018660254037844388,
+      "loss": 2.1617,
+      "step": 20
+    },
+    {
+      "epoch": 0.3105590062111801,
+      "grad_norm": 1.244139552116394,
+      "learning_rate": 0.0001737277336810124,
+      "loss": 1.7502,
+      "step": 25
+    },
+    {
+      "epoch": 0.37267080745341613,
+      "grad_norm": 0.6513166427612305,
+      "learning_rate": 0.0001573576436351046,
+      "loss": 1.457,
+      "step": 30
+    },
+    {
+      "epoch": 0.43478260869565216,
+      "grad_norm": 0.5247058272361755,
+      "learning_rate": 0.000138268343236509,
+      "loss": 1.2956,
+      "step": 35
+    },
+    {
+      "epoch": 0.4968944099378882,
+      "grad_norm": 0.3719463050365448,
+      "learning_rate": 0.00011736481776669306,
+      "loss": 1.1919,
+      "step": 40
+    },
+    {
+      "epoch": 0.5590062111801242,
+      "grad_norm": 0.31174135208129883,
+      "learning_rate": 9.563806126346642e-05,
+      "loss": 1.1399,
+      "step": 45
+    },
+    {
+      "epoch": 0.6211180124223602,
+      "grad_norm": 0.2663726508617401,
+      "learning_rate": 7.411809548974792e-05,
+      "loss": 1.1075,
+      "step": 50
+    },
+    {
+      "epoch": 0.6832298136645962,
+      "grad_norm": 0.2559656798839569,
+      "learning_rate": 5.382513867649663e-05,
+      "loss": 1.0894,
+      "step": 55
+    },
+    {
+      "epoch": 0.7453416149068323,
+      "grad_norm": 0.25429072976112366,
+      "learning_rate": 3.5721239031346066e-05,
+      "loss": 1.0851,
+      "step": 60
+    },
+    {
+      "epoch": 0.8074534161490683,
+      "grad_norm": 0.21603497862815857,
+      "learning_rate": 2.0664665970876496e-05,
+      "loss": 1.0719,
+      "step": 65
+    },
+    {
+      "epoch": 0.8695652173913043,
+      "grad_norm": 0.2722736895084381,
+      "learning_rate": 9.369221296335006e-06,
+      "loss": 1.0704,
+      "step": 70
+    },
+    {
+      "epoch": 0.9316770186335404,
+      "grad_norm": 0.22295545041561127,
+      "learning_rate": 2.3703992880066638e-06,
+      "loss": 1.0754,
+      "step": 75
+    },
+    {
+      "epoch": 0.9937888198757764,
+      "grad_norm": 0.2400663048028946,
+      "learning_rate": 0.0,
+      "loss": 1.0723,
+      "step": 80
+    },
+    {
+      "epoch": 0.9937888198757764,
+      "eval_loss": 1.4633597135543823,
+      "eval_runtime": 2.2596,
+      "eval_samples_per_second": 4.868,
+      "eval_steps_per_second": 0.443,
+      "step": 80
+    },
+    {
+      "epoch": 0.9937888198757764,
+      "step": 80,
+      "total_flos": 1.617984821877801e+18,
+      "train_loss": 1.690801477432251,
+      "train_runtime": 1102.9963,
+      "train_samples_per_second": 37.273,
+      "train_steps_per_second": 0.073
+    }
+  ],
+  "logging_steps": 5,
+  "max_steps": 80,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 100,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 1.617984821877801e+18,
+  "train_batch_size": 32,
+  "trial_name": null,
+  "trial_params": null
+}