Jessiecs/llama-2-7b-a3-1

README.md

@@ -0,0 +1,57 @@
+---
+library_name: peft
+tags:
+- generated_from_trainer
+base_model: meta-llama/Llama-2-7b-chat-hf
+model-index:
+- name: outputs
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# outputs
+
+This model is a fine-tuned version of [meta-llama/Llama-2-7b-chat-hf](https://huggingface.co/meta-llama/Llama-2-7b-chat-hf) on an unknown dataset.
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 0.0002
+- train_batch_size: 1
+- eval_batch_size: 8
+- seed: 42
+- gradient_accumulation_steps: 4
+- total_train_batch_size: 4
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: linear
+- lr_scheduler_warmup_steps: 2
+- training_steps: 20
+- mixed_precision_training: Native AMP
+
+### Training results
+
+
+
+### Framework versions
+
+- PEFT 0.8.2
+- Transformers 4.39.0.dev0
+- Pytorch 2.1.0+cu121
+- Datasets 2.17.1
+- Tokenizers 0.15.2

adapter_config.json

@@ -0,0 +1,26 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "meta-llama/Llama-2-7b-chat-hf",
+  "bias": "none",
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 64,
+  "lora_dropout": 0.1,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 16,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "base_layer"
+  ],
+  "task_type": "CAUSAL_LM",
+  "use_rslora": false
+}

adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0a4909f8c8f1fb5097329fc4cce6968ca301acdbc38a395c5621235242f73b2d
+size 319941280

all_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 0.01,
+    "total_flos": 1340174549999616.0,
+    "train_loss": 1.5259597092866897,
+    "train_runtime": 82.7118,
+    "train_samples_per_second": 0.967,
+    "train_steps_per_second": 0.242
+}

runs/Feb26_12-36-45_4abf2c950e1d/events.out.tfevents.1708951006.4abf2c950e1d.2583.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a551ac9a67f37f265c2528acdc9420c0278d6e3c3430f04044aa367d7cc4a3bc
+size 5043

runs/Feb26_12-45-13_4abf2c950e1d/events.out.tfevents.1708951513.4abf2c950e1d.2583.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b548109bffb56a39ea4d5e77050d2910265907198de8bdd44760f02fcdeb9c17
+size 9531

train_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 0.01,
+    "total_flos": 1340174549999616.0,
+    "train_loss": 1.5259597092866897,
+    "train_runtime": 82.7118,
+    "train_samples_per_second": 0.967,
+    "train_steps_per_second": 0.242
+}

trainer_state.json

@@ -0,0 +1,170 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.00812842918106076,
+  "eval_steps": 500,
+  "global_step": 20,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0,
+      "grad_norm": 1.7533113956451416,
+      "learning_rate": 0.0001,
+      "loss": 1.7094,
+      "step": 1
+    },
+    {
+      "epoch": 0.0,
+      "grad_norm": 1.4007155895233154,
+      "learning_rate": 0.0002,
+      "loss": 1.2964,
+      "step": 2
+    },
+    {
+      "epoch": 0.0,
+      "grad_norm": 1.439706563949585,
+      "learning_rate": 0.00018888888888888888,
+      "loss": 1.5479,
+      "step": 3
+    },
+    {
+      "epoch": 0.0,
+      "grad_norm": 1.9308393001556396,
+      "learning_rate": 0.00017777777777777779,
+      "loss": 1.9081,
+      "step": 4
+    },
+    {
+      "epoch": 0.0,
+      "grad_norm": 1.1223907470703125,
+      "learning_rate": 0.0001666666666666667,
+      "loss": 1.7825,
+      "step": 5
+    },
+    {
+      "epoch": 0.0,
+      "grad_norm": 0.7965916395187378,
+      "learning_rate": 0.00015555555555555556,
+      "loss": 1.0745,
+      "step": 6
+    },
+    {
+      "epoch": 0.0,
+      "grad_norm": 0.8971462249755859,
+      "learning_rate": 0.00014444444444444444,
+      "loss": 0.9313,
+      "step": 7
+    },
+    {
+      "epoch": 0.0,
+      "grad_norm": 1.9314714670181274,
+      "learning_rate": 0.00013333333333333334,
+      "loss": 1.6707,
+      "step": 8
+    },
+    {
+      "epoch": 0.0,
+      "grad_norm": 1.2980624437332153,
+      "learning_rate": 0.00012222222222222224,
+      "loss": 1.8406,
+      "step": 9
+    },
+    {
+      "epoch": 0.0,
+      "grad_norm": 1.1596261262893677,
+      "learning_rate": 0.00011111111111111112,
+      "loss": 1.4502,
+      "step": 10
+    },
+    {
+      "epoch": 0.0,
+      "grad_norm": 0.9083566069602966,
+      "learning_rate": 0.0001,
+      "loss": 1.3373,
+      "step": 11
+    },
+    {
+      "epoch": 0.0,
+      "grad_norm": 1.6125576496124268,
+      "learning_rate": 8.888888888888889e-05,
+      "loss": 1.4743,
+      "step": 12
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 1.1022965908050537,
+      "learning_rate": 7.777777777777778e-05,
+      "loss": 1.4054,
+      "step": 13
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 1.1945298910140991,
+      "learning_rate": 6.666666666666667e-05,
+      "loss": 1.8368,
+      "step": 14
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 4.051852226257324,
+      "learning_rate": 5.555555555555556e-05,
+      "loss": 1.7753,
+      "step": 15
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 1.8756366968154907,
+      "learning_rate": 4.4444444444444447e-05,
+      "loss": 1.2103,
+      "step": 16
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 1.6548067331314087,
+      "learning_rate": 3.3333333333333335e-05,
+      "loss": 1.5789,
+      "step": 17
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 1.1712427139282227,
+      "learning_rate": 2.2222222222222223e-05,
+      "loss": 1.2315,
+      "step": 18
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 1.0134000778198242,
+      "learning_rate": 1.1111111111111112e-05,
+      "loss": 1.9206,
+      "step": 19
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 1.2952359914779663,
+      "learning_rate": 0.0,
+      "loss": 1.5372,
+      "step": 20
+    },
+    {
+      "epoch": 0.01,
+      "step": 20,
+      "total_flos": 1340174549999616.0,
+      "train_loss": 1.5259597092866897,
+      "train_runtime": 82.7118,
+      "train_samples_per_second": 0.967,
+      "train_steps_per_second": 0.242
+    }
+  ],
+  "logging_steps": 1,
+  "max_steps": 20,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 500,
+  "total_flos": 1340174549999616.0,
+  "train_batch_size": 1,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aadb207f52156693078ce1082726855ac72e416bfa3799f3104b1f08fd2396c2
+size 4920