Upload folder using huggingface_hub

README.md

@@ -199,4 +199,4 @@ Carbon emissions can be estimated using the [Machine Learning Impact calculator]
 [More Information Needed]
 ### Framework versions
 
-- PEFT 0.14.0
+- PEFT 0.12.0

adapter_config.json

@@ -3,8 +3,6 @@
   "auto_mapping": null,
   "base_model_name_or_path": "microsoft/Phi-3.5-mini-instruct",
   "bias": "none",
-  "eva_config": null,
-  "exclude_modules": null,
   "fan_in_fan_out": false,
   "inference_mode": true,
   "init_lora_weights": true,
@@ -12,20 +10,19 @@
   "layers_pattern": null,
   "layers_to_transform": null,
   "loftq_config": {},
-  "lora_alpha": 64,
-  "lora_bias": false,
-  "lora_dropout": 0.0001,
+  "lora_alpha": 16,
+  "lora_dropout": 0.05,
   "megatron_config": null,
   "megatron_core": "megatron.core",
   "modules_to_save": null,
   "peft_type": "LORA",
-  "r": 32,
+  "r": 8,
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
-    "o_proj",
-    "qkv_proj",
     "down_proj",
+    "qkv_proj",
+    "o_proj",
     "gate_up_proj"
   ],
   "task_type": "CAUSAL_LM",

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:722e0d11c18df2484130fccf458fb994d57d4dee4423f6ff233ab20595ac5492
-size 201361312
+oid sha256:f038e0934a3168a134372d76747c35963d810535ac44238c96f526ba5d73444d
+size 50365768

added_tokens.json

@@ -2,6 +2,7 @@
   "<|assistant|>": 32001,
   "<|endoftext|>": 32000,
   "<|end|>": 32007,
+  "<|pad|>": 32011,
   "<|placeholder1|>": 32002,
   "<|placeholder2|>": 32003,
   "<|placeholder3|>": 32004,

optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9ef9fd6ce3475d48ab32d6a7526e8f236b02b149d6e84fd712103775f348a974
-size 402868986
+oid sha256:31c2c06a3aa791be14685b5d59899a877c559219322620a519bd1b84ef483910
+size 100878458

rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:dde5f077a5393538b5aed42dc077f2c26a3ecb3009a6cd8323a0963c172eeafe
+oid sha256:936b713051d3954452ec3bf4371217942ece2f8826b34ec15d76739514f6eb2c
 size 14244

scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a16bc59ca2ad7c9d866e071897b87e0c2309c5def808c0078c92b1caa75df182
+oid sha256:a492b6d65e6e851d97e2025279f088a6a581867a0b4272350b086ae52aef4d06
 size 1064

special_tokens_map.json

@@ -7,14 +7,14 @@
     "single_word": false
   },
   "eos_token": {
-    "content": "<|endoftext|>",
+    "content": "<|end|>",
     "lstrip": false,
     "normalized": false,
     "rstrip": false,
     "single_word": false
   },
   "pad_token": {
-    "content": "<|endoftext|>",
+    "content": "<|pad|>",
     "lstrip": false,
     "normalized": false,
     "rstrip": false,

tokenizer.json

@@ -1,7 +1,21 @@
 {
   "version": "1.0",
-  "truncation": null,
-  "padding": null,
+  "truncation": {
+    "direction": "Right",
+    "max_length": 2048,
+    "strategy": "LongestFirst",
+    "stride": 0
+  },
+  "padding": {
+    "strategy": {
+      "Fixed": 2048
+    },
+    "direction": "Left",
+    "pad_to_multiple_of": null,
+    "pad_id": 32011,
+    "pad_type_id": 0,
+    "pad_token": "<|pad|>"
+  },
   "added_tokens": [
     {
       "id": 0,
@@ -98,7 +112,7 @@
       "content": "<|end|>",
       "single_word": false,
       "lstrip": false,
-      "rstrip": true,
+      "rstrip": false,
       "normalized": false,
       "special": true
     },
@@ -128,6 +142,15 @@
       "rstrip": true,
       "normalized": false,
       "special": true
+    },
+    {
+      "id": 32011,
+      "content": "<|pad|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
     }
   ],
   "normalizer": {

tokenizer_config.json

@@ -87,7 +87,7 @@
       "content": "<|end|>",
       "lstrip": false,
       "normalized": false,
-      "rstrip": true,
+      "rstrip": false,
       "single_word": false,
       "special": true
     },
@@ -114,16 +114,24 @@
       "rstrip": true,
       "single_word": false,
       "special": true
+    },
+    "32011": {
+      "content": "<|pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
     }
   },
   "bos_token": "<s>",
   "chat_template": "{% for message in messages %}{% if message['role'] == 'system' and message['content'] %}{{'<|system|>\n' + message['content'] + '<|end|>\n'}}{% elif message['role'] == 'user' %}{{'<|user|>\n' + message['content'] + '<|end|>\n'}}{% elif message['role'] == 'assistant' %}{{'<|assistant|>\n' + message['content'] + '<|end|>\n'}}{% endif %}{% endfor %}{% if add_generation_prompt %}{{ '<|assistant|>\n' }}{% else %}{{ eos_token }}{% endif %}",
   "clean_up_tokenization_spaces": false,
-  "eos_token": "<|endoftext|>",
+  "eos_token": "<|end|>",
   "extra_special_tokens": {},
   "legacy": false,
   "model_max_length": 131072,
-  "pad_token": "<|endoftext|>",
+  "pad_token": "<|pad|>",
   "padding_side": "left",
   "sp_model_kwargs": {},
   "tokenizer_class": "LlamaTokenizer",

trainer_state.json

@@ -1,240 +1,17 @@
 {
-  "best_global_step": 99,
-  "best_metric": 0.6646606922149658,
-  "best_model_checkpoint": "//outputs/task7_microsoft/Phi-3.5-mini-instruct/checkpoint-99",
-  "epoch": 11.0,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.9481481481481482,
   "eval_steps": 500,
-  "global_step": 99,
+  "global_step": 66,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
-  "log_history": [
-    {
-      "epoch": 0.5882352941176471,
-      "grad_norm": 0.5897260308265686,
-      "learning_rate": 1e-05,
-      "loss": 0.8043,
-      "step": 5
-    },
-    {
-      "epoch": 1.0,
-      "eval_loss": 0.7620137929916382,
-      "eval_runtime": 3.3723,
-      "eval_samples_per_second": 4.448,
-      "eval_steps_per_second": 0.593,
-      "step": 9
-    },
-    {
-      "epoch": 1.1176470588235294,
-      "grad_norm": 0.5304206609725952,
-      "learning_rate": 9.931806517013612e-06,
-      "loss": 0.7621,
-      "step": 10
-    },
-    {
-      "epoch": 1.7058823529411766,
-      "grad_norm": 0.3766116499900818,
-      "learning_rate": 9.729086208503174e-06,
-      "loss": 0.7587,
-      "step": 15
-    },
-    {
-      "epoch": 2.0,
-      "eval_loss": 0.7274001240730286,
-      "eval_runtime": 3.3712,
-      "eval_samples_per_second": 4.449,
-      "eval_steps_per_second": 0.593,
-      "step": 18
-    },
-    {
-      "epoch": 2.235294117647059,
-      "grad_norm": 0.2997392416000366,
-      "learning_rate": 9.397368756032445e-06,
-      "loss": 0.7073,
-      "step": 20
-    },
-    {
-      "epoch": 2.8235294117647056,
-      "grad_norm": 0.2789791226387024,
-      "learning_rate": 8.94570254698197e-06,
-      "loss": 0.6531,
-      "step": 25
-    },
-    {
-      "epoch": 3.0,
-      "eval_loss": 0.7077590823173523,
-      "eval_runtime": 3.3715,
-      "eval_samples_per_second": 4.449,
-      "eval_steps_per_second": 0.593,
-      "step": 27
-    },
-    {
-      "epoch": 3.3529411764705883,
-      "grad_norm": 0.28947126865386963,
-      "learning_rate": 8.386407858128707e-06,
-      "loss": 0.7029,
-      "step": 30
-    },
-    {
-      "epoch": 3.9411764705882355,
-      "grad_norm": 0.2775668799877167,
-      "learning_rate": 7.734740790612137e-06,
-      "loss": 0.6033,
-      "step": 35
-    },
-    {
-      "epoch": 4.0,
-      "eval_loss": 0.6937279105186462,
-      "eval_runtime": 3.3722,
-      "eval_samples_per_second": 4.448,
-      "eval_steps_per_second": 0.593,
-      "step": 36
-    },
-    {
-      "epoch": 4.470588235294118,
-      "grad_norm": 0.24069756269454956,
-      "learning_rate": 7.008477123264849e-06,
-      "loss": 0.6549,
-      "step": 40
-    },
-    {
-      "epoch": 5.0,
-      "grad_norm": 0.31259897351264954,
-      "learning_rate": 6.227427435703997e-06,
-      "loss": 0.694,
-      "step": 45
-    },
-    {
-      "epoch": 5.0,
-      "eval_loss": 0.6824610829353333,
-      "eval_runtime": 3.3719,
-      "eval_samples_per_second": 4.449,
-      "eval_steps_per_second": 0.593,
-      "step": 45
-    },
-    {
-      "epoch": 5.588235294117647,
-      "grad_norm": 0.25407281517982483,
-      "learning_rate": 5.412896727361663e-06,
-      "loss": 0.6865,
-      "step": 50
-    },
-    {
-      "epoch": 6.0,
-      "eval_loss": 0.6763660311698914,
-      "eval_runtime": 3.3717,
-      "eval_samples_per_second": 4.449,
-      "eval_steps_per_second": 0.593,
-      "step": 54
-    },
-    {
-      "epoch": 6.117647058823529,
-      "grad_norm": 0.2767919600009918,
-      "learning_rate": 4.587103272638339e-06,
-      "loss": 0.6081,
-      "step": 55
-    },
-    {
-      "epoch": 6.705882352941177,
-      "grad_norm": 0.24282197654247284,
-      "learning_rate": 3.7725725642960047e-06,
-      "loss": 0.6577,
-      "step": 60
-    },
-    {
-      "epoch": 7.0,
-      "eval_loss": 0.6713435649871826,
-      "eval_runtime": 3.3715,
-      "eval_samples_per_second": 4.449,
-      "eval_steps_per_second": 0.593,
-      "step": 63
-    },
-    {
-      "epoch": 7.235294117647059,
-      "grad_norm": 0.1744387447834015,
-      "learning_rate": 2.991522876735154e-06,
-      "loss": 0.5941,
-      "step": 65
-    },
-    {
-      "epoch": 7.823529411764706,
-      "grad_norm": 0.20212271809577942,
-      "learning_rate": 2.265259209387867e-06,
-      "loss": 0.6509,
-      "step": 70
-    },
-    {
-      "epoch": 8.0,
-      "eval_loss": 0.6677358150482178,
-      "eval_runtime": 3.3723,
-      "eval_samples_per_second": 4.448,
-      "eval_steps_per_second": 0.593,
-      "step": 72
-    },
-    {
-      "epoch": 8.352941176470589,
-      "grad_norm": 0.16120634973049164,
-      "learning_rate": 1.6135921418712959e-06,
-      "loss": 0.5923,
-      "step": 75
-    },
-    {
-      "epoch": 8.941176470588236,
-      "grad_norm": 0.2318679690361023,
-      "learning_rate": 1.0542974530180327e-06,
-      "loss": 0.6438,
-      "step": 80
-    },
-    {
-      "epoch": 9.0,
-      "eval_loss": 0.6655252575874329,
-      "eval_runtime": 3.3713,
-      "eval_samples_per_second": 4.449,
-      "eval_steps_per_second": 0.593,
-      "step": 81
-    },
-    {
-      "epoch": 9.470588235294118,
-      "grad_norm": 0.19574101269245148,
-      "learning_rate": 6.026312439675553e-07,
-      "loss": 0.6359,
-      "step": 85
-    },
-    {
-      "epoch": 10.0,
-      "grad_norm": 0.22642117738723755,
-      "learning_rate": 2.7091379149682683e-07,
-      "loss": 0.5741,
-      "step": 90
-    },
-    {
-      "epoch": 10.0,
-      "eval_loss": 0.6652756929397583,
-      "eval_runtime": 3.3709,
-      "eval_samples_per_second": 4.45,
-      "eval_steps_per_second": 0.593,
-      "step": 90
-    },
-    {
-      "epoch": 10.588235294117647,
-      "grad_norm": 0.2666153013706207,
-      "learning_rate": 6.819348298638839e-08,
-      "loss": 0.6734,
-      "step": 95
-    },
-    {
-      "epoch": 11.0,
-      "eval_loss": 0.6646606922149658,
-      "eval_runtime": 3.3717,
-      "eval_samples_per_second": 4.449,
-      "eval_steps_per_second": 0.593,
-      "step": 99
-    }
-  ],
-  "logging_steps": 5,
-  "max_steps": 100,
+  "log_history": [],
+  "logging_steps": 100,
+  "max_steps": 66,
   "num_input_tokens_seen": 0,
-  "num_train_epochs": 13,
+  "num_train_epochs": 2,
   "save_steps": 500,
   "stateful_callbacks": {
     "TrainerControl": {
@@ -243,13 +20,13 @@
         "should_evaluate": false,
         "should_log": false,
         "should_save": true,
-        "should_training_stop": false
+        "should_training_stop": true
       },
       "attributes": {}
     }
   },
-  "total_flos": 5.180655973758566e+16,
-  "train_batch_size": 2,
+  "total_flos": 1.2071086917156864e+16,
+  "train_batch_size": 1,
   "trial_name": null,
   "trial_params": null
 }

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:02dca0ceed349b196a13d4f3de83bdc1d637c3f4a599aaf1cc66d5744a87d6c3
-size 5624
+oid sha256:1cd9f76f1c27b8546e1ede6a4353769298e7cfd4ca0d040b456617a994550ac6
+size 5304