Training in progress, step 100, checkpoint

last-checkpoint/README.md

@@ -1,12 +1,6 @@
 ---
 base_model: mistralai/Mistral-7B-Instruct-v0.2
 library_name: peft
-pipeline_tag: text-generation
-tags:
-- base_model:adapter:mistralai/Mistral-7B-Instruct-v0.2
-- lora
-- sft
-- trl
 ---
 
 # Model Card for Model ID
@@ -205,4 +199,4 @@ Carbon emissions can be estimated using the [Machine Learning Impact calculator]
 [More Information Needed]
 ### Framework versions
 
-- PEFT 0.19.1
+- PEFT 0.13.0

last-checkpoint/adapter_config.json

@@ -1,14 +1,8 @@
 {
-  "alora_invocation_tokens": null,
   "alpha_pattern": {},
-  "arrow_config": null,
   "auto_mapping": null,
-  "base_model_name_or_path": null,
+  "base_model_name_or_path": "mistralai/Mistral-7B-Instruct-v0.2",
   "bias": "none",
-  "corda_config": null,
-  "ensure_weight_tying": false,
-  "eva_config": null,
-  "exclude_modules": null,
   "fan_in_fan_out": false,
   "inference_mode": true,
   "init_lora_weights": true,
@@ -17,32 +11,24 @@
   "layers_to_transform": null,
   "loftq_config": {},
   "lora_alpha": 32,
-  "lora_bias": false,
   "lora_dropout": 0.05,
-  "lora_ga_config": null,
   "megatron_config": null,
   "megatron_core": "megatron.core",
   "modules_to_save": null,
   "peft_type": "LORA",
-  "peft_version": "0.19.1",
-  "qalora_group_size": 16,
   "r": 16,
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
-    "q_proj",
+    "o_proj",
+    "down_proj",
     "gate_proj",
-    "up_proj",
-    "k_proj",
+    "q_proj",
     "v_proj",
-    "down_proj",
-    "o_proj"
+    "k_proj",
+    "up_proj"
   ],
-  "target_parameters": null,
   "task_type": "CAUSAL_LM",
-  "trainable_token_indices": null,
-  "use_bdlora": null,
   "use_dora": false,
-  "use_qalora": false,
   "use_rslora": false
 }

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e0c28ae9fda7ed077120f5397e44e06d2691b2ea2ada8010ba232ded303bcf15
-size 83953808
+oid sha256:9fe43da4e97eebeaa0158b36a25c885b64c459b8e04460cfe58ff057a1a9e33d
+size 83945744

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3f9ebec1e37b3d5f04a61418f5f4316e15169a3a32f9b73632b74e8003b53ca8
-size 85733925
+oid sha256:481d0894db7c13052290caab3e52d3e88de5e5cdf4ec83a8110888cb9d8eb75e
+size 85728997

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:768b685cf09f245ca09014d59140a335cf62bc70a754dee586b97004fcdb04c8
+oid sha256:5c489d94a3653edcd1381871cbb21ee09c7242857160be9a015191844079d14a
 size 14645

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:19476b10a067a1a5864bfeae7a86583ec2326a22d2ca0845e094959744d7ef6d
+oid sha256:31dcc952e23cde9c32791f5eef02f3ae04c11b4696574b49b612390af96bc354
 size 1465

last-checkpoint/special_tokens_map.json

@@ -0,0 +1,24 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "</s>",
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

last-checkpoint/tokenizer.json

@@ -1,6 +1,11 @@
 {
   "version": "1.0",
-  "truncation": null,
+  "truncation": {
+    "direction": "Right",
+    "max_length": 1024,
+    "strategy": "LongestFirst",
+    "stride": 0
+  },
   "padding": null,
   "added_tokens": [
     {

last-checkpoint/tokenizer_config.json

@@ -1,19 +1,45 @@
 {
+  "add_bos_token": true,
+  "add_eos_token": false,
   "add_prefix_space": null,
-  "backend": "tokenizers",
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [],
   "bos_token": "<s>",
+  "chat_template": "{%- if messages[0]['role'] == 'system' %}\n    {%- set system_message = messages[0]['content'] %}\n    {%- set loop_messages = messages[1:] %}\n{%- else %}\n    {%- set loop_messages = messages %}\n{%- endif %}\n\n{{- bos_token }}\n{%- for message in loop_messages %}\n    {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}\n        {{- raise_exception('After the optional system message, conversation roles must alternate user/assistant/user/assistant/...') }}\n    {%- endif %}\n    {%- if message['role'] == 'user' %}\n        {%- if loop.first and system_message is defined %}\n            {{- ' [INST] ' + system_message + '\\n\\n' + message['content'] + ' [/INST]' }}\n        {%- else %}\n            {{- ' [INST] ' + message['content'] + ' [/INST]' }}\n        {%- endif %}\n    {%- elif message['role'] == 'assistant' %}\n        {{- ' ' + message['content'] + eos_token}}\n    {%- else %}\n        {{- raise_exception('Only user and assistant roles are supported, with the exception of an initial optional system message!') }}\n    {%- endif %}\n{%- endfor %}\n",
   "clean_up_tokenization_spaces": false,
   "eos_token": "</s>",
-  "extra_special_tokens": [],
-  "is_local": false,
   "legacy": false,
-  "local_files_only": false,
   "model_max_length": 1000000000000000019884624838656,
   "pad_token": "</s>",
   "padding_side": "right",
   "sp_model_kwargs": {},
   "spaces_between_special_tokens": false,
-  "tokenizer_class": "TokenizersBackend",
+  "tokenizer_class": "LlamaTokenizer",
   "unk_token": "<unk>",
   "use_default_system_prompt": false
 }

last-checkpoint/trainer_state.json

@@ -1,425 +1,52 @@
 {
-  "best_global_step": 800,
-  "best_metric": 0.6767381429672241,
-  "best_model_checkpoint": "./learnhub-checkpoints/checkpoint-800",
-  "epoch": 0.6349836293283064,
+  "best_metric": 0.8977340459823608,
+  "best_model_checkpoint": "./learnhub-checkpoints/checkpoint-100",
+  "epoch": 0.0793729536660383,
   "eval_steps": 100,
-  "global_step": 800,
+  "global_step": 100,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
-      "entropy": 1.5423119881749152,
       "epoch": 0.019843238416509576,
-      "grad_norm": 2.125,
-      "learning_rate": 4.8e-05,
-      "loss": 2.1809950256347657,
-      "mean_token_accuracy": 0.5977656890451908,
-      "num_tokens": 42575.0,
+      "grad_norm": 18.390625,
+      "learning_rate": 5e-05,
+      "loss": 16.5658,
       "step": 25
     },
     {
-      "entropy": 1.1193951192498206,
       "epoch": 0.03968647683301915,
-      "grad_norm": 1.671875,
-      "learning_rate": 9.8e-05,
-      "loss": 1.1683277893066406,
-      "mean_token_accuracy": 0.7463910706341267,
-      "num_tokens": 85122.0,
+      "grad_norm": 15.34375,
+      "learning_rate": 0.0001,
+      "loss": 9.0596,
       "step": 50
     },
     {
-      "entropy": 0.9514059691131115,
       "epoch": 0.05952971524952872,
-      "grad_norm": 1.6328125,
-      "learning_rate": 0.000148,
-      "loss": 0.9270614624023438,
-      "mean_token_accuracy": 0.7785248437523842,
-      "num_tokens": 131421.0,
+      "grad_norm": 12.78125,
+      "learning_rate": 0.00015000000000000001,
+      "loss": 7.4307,
       "step": 75
     },
     {
-      "entropy": 0.9425666096806526,
       "epoch": 0.0793729536660383,
-      "grad_norm": 0.84765625,
-      "learning_rate": 0.00019800000000000002,
-      "loss": 0.9538876342773438,
-      "mean_token_accuracy": 0.7819291327893734,
-      "num_tokens": 176055.0,
+      "grad_norm": 9.1796875,
+      "learning_rate": 0.0002,
+      "loss": 7.2264,
       "step": 100
     },
     {
       "epoch": 0.0793729536660383,
-      "eval_entropy": 0.9276722411988145,
-      "eval_loss": 0.8972787261009216,
-      "eval_mean_token_accuracy": 0.7821191083464469,
-      "eval_num_tokens": 176055.0,
-      "eval_runtime": 290.3105,
-      "eval_samples_per_second": 3.655,
-      "eval_steps_per_second": 1.829,
+      "eval_loss": 0.8977340459823608,
+      "eval_runtime": 789.8478,
+      "eval_samples_per_second": 1.343,
+      "eval_steps_per_second": 0.672,
       "step": 100
-    },
-    {
-      "entropy": 0.8993638175725936,
-      "epoch": 0.09921619208254787,
-      "grad_norm": 1.453125,
-      "learning_rate": 0.00019997901149992398,
-      "loss": 0.9062361145019531,
-      "mean_token_accuracy": 0.7870219559967517,
-      "num_tokens": 218572.0,
-      "step": 125
-    },
-    {
-      "entropy": 0.8412108091264963,
-      "epoch": 0.11905943049905744,
-      "grad_norm": 1.2578125,
-      "learning_rate": 0.00019991252117460662,
-      "loss": 0.8404283905029297,
-      "mean_token_accuracy": 0.7994147537648678,
-      "num_tokens": 264473.0,
-      "step": 150
-    },
-    {
-      "entropy": 0.8592079882323742,
-      "epoch": 0.13890266891556702,
-      "grad_norm": 1.34375,
-      "learning_rate": 0.00019980052286930407,
-      "loss": 0.8845314025878906,
-      "mean_token_accuracy": 0.7938638082146645,
-      "num_tokens": 308986.0,
-      "step": 175
-    },
-    {
-      "entropy": 0.8827662563323975,
-      "epoch": 0.1587459073320766,
-      "grad_norm": 1.4453125,
-      "learning_rate": 0.000199643067596808,
-      "loss": 0.9247197723388672,
-      "mean_token_accuracy": 0.7900393509864807,
-      "num_tokens": 353705.0,
-      "step": 200
-    },
-    {
-      "epoch": 0.1587459073320766,
-      "eval_entropy": 0.8343640095545523,
-      "eval_loss": 0.815721869468689,
-      "eval_mean_token_accuracy": 0.7949616669935021,
-      "eval_num_tokens": 353705.0,
-      "eval_runtime": 290.3166,
-      "eval_samples_per_second": 3.655,
-      "eval_steps_per_second": 1.829,
-      "step": 200
-    },
-    {
-      "entropy": 0.8292603352665902,
-      "epoch": 0.17858914574858617,
-      "grad_norm": 1.3828125,
-      "learning_rate": 0.00019944022707456992,
-      "loss": 0.8381348419189453,
-      "mean_token_accuracy": 0.8005190336704254,
-      "num_tokens": 396464.0,
-      "step": 225
-    },
-    {
-      "entropy": 0.8279347644746303,
-      "epoch": 0.19843238416509573,
-      "grad_norm": 1.3671875,
-      "learning_rate": 0.00019919209369203533,
-      "loss": 0.8449878692626953,
-      "mean_token_accuracy": 0.7975908493995667,
-      "num_tokens": 443396.0,
-      "step": 250
-    },
-    {
-      "entropy": 0.847797994017601,
-      "epoch": 0.21827562258160532,
-      "grad_norm": 1.40625,
-      "learning_rate": 0.0001988987804685623,
-      "loss": 0.7986299896240234,
-      "mean_token_accuracy": 0.7965799477696419,
-      "num_tokens": 488198.0,
-      "step": 275
-    },
-    {
-      "entropy": 0.8216452768445015,
-      "epoch": 0.23811886099811488,
-      "grad_norm": 1.109375,
-      "learning_rate": 0.00019856042100194356,
-      "loss": 0.785276870727539,
-      "mean_token_accuracy": 0.8020041480660438,
-      "num_tokens": 535664.0,
-      "step": 300
-    },
-    {
-      "epoch": 0.23811886099811488,
-      "eval_entropy": 0.8421865575254076,
-      "eval_loss": 0.7751030921936035,
-      "eval_mean_token_accuracy": 0.8036603897304858,
-      "eval_num_tokens": 535664.0,
-      "eval_runtime": 291.1861,
-      "eval_samples_per_second": 3.644,
-      "eval_steps_per_second": 1.824,
-      "step": 300
-    },
-    {
-      "entropy": 0.7873362612724304,
-      "epoch": 0.25796209941462445,
-      "grad_norm": 1.1640625,
-      "learning_rate": 0.00019817716940755586,
-      "loss": 0.7424690246582031,
-      "mean_token_accuracy": 0.8078745475411415,
-      "num_tokens": 581174.0,
-      "step": 325
-    },
-    {
-      "entropy": 0.7965878197550773,
-      "epoch": 0.27780533783113404,
-      "grad_norm": 1.46875,
-      "learning_rate": 0.00019774920024816353,
-      "loss": 0.7590773773193359,
-      "mean_token_accuracy": 0.8090151616930962,
-      "num_tokens": 623047.0,
-      "step": 350
-    },
-    {
-      "entropy": 0.7574200442433358,
-      "epoch": 0.2976485762476436,
-      "grad_norm": 1.3515625,
-      "learning_rate": 0.00019727670845440893,
-      "loss": 0.7333896636962891,
-      "mean_token_accuracy": 0.8140154486894607,
-      "num_tokens": 665332.0,
-      "step": 375
-    },
-    {
-      "entropy": 0.7607150036841631,
-      "epoch": 0.3174918146641532,
-      "grad_norm": 1.3671875,
-      "learning_rate": 0.00019675990923602598,
-      "loss": 0.7356826019287109,
-      "mean_token_accuracy": 0.8127754744887352,
-      "num_tokens": 709558.0,
-      "step": 400
-    },
-    {
-      "epoch": 0.3174918146641532,
-      "eval_entropy": 0.7901154234865067,
-      "eval_loss": 0.7394365072250366,
-      "eval_mean_token_accuracy": 0.8106958746012097,
-      "eval_num_tokens": 709558.0,
-      "eval_runtime": 290.9403,
-      "eval_samples_per_second": 3.647,
-      "eval_steps_per_second": 1.825,
-      "step": 400
-    },
-    {
-      "entropy": 0.7346216081827879,
-      "epoch": 0.33733505308066275,
-      "grad_norm": 1.46875,
-      "learning_rate": 0.0001961990379838167,
-      "loss": 0.6551874542236328,
-      "mean_token_accuracy": 0.8211761102080345,
-      "num_tokens": 755393.0,
-      "step": 425
-    },
-    {
-      "entropy": 0.7677556264400482,
-      "epoch": 0.35717829149717234,
-      "grad_norm": 1.7421875,
-      "learning_rate": 0.0001955943501624357,
-      "loss": 0.7343796539306641,
-      "mean_token_accuracy": 0.812620207965374,
-      "num_tokens": 800020.0,
-      "step": 450
-    },
-    {
-      "entropy": 0.7691158069670201,
-      "epoch": 0.37702152991368193,
-      "grad_norm": 1.2734375,
-      "learning_rate": 0.00019494612119403177,
-      "loss": 0.7420792388916015,
-      "mean_token_accuracy": 0.8088723468780518,
-      "num_tokens": 842459.0,
-      "step": 475
-    },
-    {
-      "entropy": 0.7609671781212092,
-      "epoch": 0.39686476833019146,
-      "grad_norm": 1.28125,
-      "learning_rate": 0.00019425464633279906,
-      "loss": 0.733401870727539,
-      "mean_token_accuracy": 0.8129055750370026,
-      "num_tokens": 884620.0,
-      "step": 500
-    },
-    {
-      "epoch": 0.39686476833019146,
-      "eval_entropy": 0.7718784739938609,
-      "eval_loss": 0.7210129499435425,
-      "eval_mean_token_accuracy": 0.8150942843063628,
-      "eval_num_tokens": 884620.0,
-      "eval_runtime": 290.0297,
-      "eval_samples_per_second": 3.658,
-      "eval_steps_per_second": 1.831,
-      "step": 500
-    },
-    {
-      "entropy": 0.7566675854474306,
-      "epoch": 0.41670800674670105,
-      "grad_norm": 1.359375,
-      "learning_rate": 0.0001935202405304951,
-      "loss": 0.7321968078613281,
-      "mean_token_accuracy": 0.8134541392326355,
-      "num_tokens": 929259.0,
-      "step": 525
-    },
-    {
-      "entropy": 0.7571731075644493,
-      "epoch": 0.43655124516321064,
-      "grad_norm": 1.640625,
-      "learning_rate": 0.0001927432382929872,
-      "loss": 0.7315776824951172,
-      "mean_token_accuracy": 0.816252943277359,
-      "num_tokens": 974531.0,
-      "step": 550
-    },
-    {
-      "entropy": 0.7301681135594845,
-      "epoch": 0.45639448357972023,
-      "grad_norm": 10.0625,
-      "learning_rate": 0.00019192399352789232,
-      "loss": 0.7157851409912109,
-      "mean_token_accuracy": 0.8203792923688888,
-      "num_tokens": 1019376.0,
-      "step": 575
-    },
-    {
-      "entropy": 0.7234990952163934,
-      "epoch": 0.47623772199622977,
-      "grad_norm": 1.5703125,
-      "learning_rate": 0.00019106287938337984,
-      "loss": 0.6780443572998047,
-      "mean_token_accuracy": 0.818044265806675,
-      "num_tokens": 1065482.0,
-      "step": 600
-    },
-    {
-      "epoch": 0.47623772199622977,
-      "eval_entropy": 0.7563102317170683,
-      "eval_loss": 0.7063755989074707,
-      "eval_mean_token_accuracy": 0.8184236252150742,
-      "eval_num_tokens": 1065482.0,
-      "eval_runtime": 292.1942,
-      "eval_samples_per_second": 3.631,
-      "eval_steps_per_second": 1.817,
-      "step": 600
-    },
-    {
-      "entropy": 0.7022078443691134,
-      "epoch": 0.49608096041273936,
-      "grad_norm": 1.375,
-      "learning_rate": 0.00019016028807821065,
-      "loss": 0.65089599609375,
-      "mean_token_accuracy": 0.8246574628353119,
-      "num_tokens": 1113578.0,
-      "step": 625
-    },
-    {
-      "entropy": 0.7365065434575081,
-      "epoch": 0.5159241988292489,
-      "grad_norm": 1.2578125,
-      "learning_rate": 0.00018921663072309007,
-      "loss": 0.7072123718261719,
-      "mean_token_accuracy": 0.8180231443047523,
-      "num_tokens": 1158454.0,
-      "step": 650
-    },
-    {
-      "entropy": 0.6983346920460463,
-      "epoch": 0.5357674372457585,
-      "grad_norm": 1.0625,
-      "learning_rate": 0.0001882323371334159,
-      "loss": 0.6436396789550781,
-      "mean_token_accuracy": 0.8268389776349068,
-      "num_tokens": 1206571.0,
-      "step": 675
-    },
-    {
-      "entropy": 0.7212439847737551,
-      "epoch": 0.5556106756622681,
-      "grad_norm": 1.1796875,
-      "learning_rate": 0.00018720785563350667,
-      "loss": 0.6770188903808594,
-      "mean_token_accuracy": 0.8182832631468773,
-      "num_tokens": 1249976.0,
-      "step": 700
-    },
-    {
-      "epoch": 0.5556106756622681,
-      "eval_entropy": 0.7183699383910767,
-      "eval_loss": 0.6925566792488098,
-      "eval_mean_token_accuracy": 0.8202201184130883,
-      "eval_num_tokens": 1249976.0,
-      "eval_runtime": 291.4532,
-      "eval_samples_per_second": 3.64,
-      "eval_steps_per_second": 1.822,
-      "step": 700
-    },
-    {
-      "entropy": 0.7322280709445477,
-      "epoch": 0.5754539140787777,
-      "grad_norm": 1.65625,
-      "learning_rate": 0.0001861436528524,
-      "loss": 0.7028071594238281,
-      "mean_token_accuracy": 0.8174135899543762,
-      "num_tokens": 1295378.0,
-      "step": 725
-    },
-    {
-      "entropy": 0.7073389308154583,
-      "epoch": 0.5952971524952873,
-      "grad_norm": 1.40625,
-      "learning_rate": 0.00018504021351131296,
-      "loss": 0.684170913696289,
-      "mean_token_accuracy": 0.8211757111549377,
-      "num_tokens": 1338843.0,
-      "step": 750
-    },
-    {
-      "entropy": 0.705704356059432,
-      "epoch": 0.6151403909117968,
-      "grad_norm": 1.4375,
-      "learning_rate": 0.00018389804020286206,
-      "loss": 0.6670877838134766,
-      "mean_token_accuracy": 0.8235558214783668,
-      "num_tokens": 1384327.0,
-      "step": 775
-    },
-    {
-      "entropy": 0.6878117263317108,
-      "epoch": 0.6349836293283064,
-      "grad_norm": 1.53125,
-      "learning_rate": 0.00018271765316214356,
-      "loss": 0.6584226989746094,
-      "mean_token_accuracy": 0.8270227089524269,
-      "num_tokens": 1427944.0,
-      "step": 800
-    },
-    {
-      "epoch": 0.6349836293283064,
-      "eval_entropy": 0.6733546478712401,
-      "eval_loss": 0.6767381429672241,
-      "eval_mean_token_accuracy": 0.8240371050134216,
-      "eval_num_tokens": 1427944.0,
-      "eval_runtime": 290.3688,
-      "eval_samples_per_second": 3.654,
-      "eval_steps_per_second": 1.829,
-      "step": 800
     }
   ],
   "logging_steps": 25,
-  "max_steps": 3780,
+  "max_steps": 3777,
   "num_input_tokens_seen": 0,
   "num_train_epochs": 3,
   "save_steps": 100,
@@ -435,7 +62,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 7.968632183621222e+16,
+  "total_flos": 9601168072753152.0,
   "train_batch_size": 2,
   "trial_name": null,
   "trial_params": null

last-checkpoint/training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a7636bfd15768feeae5b8b0ea6c6ea570e3bbf66f8df8000ffdf82d9d44941a2
-size 5777
+oid sha256:8306ac097284337a989527ee29aa84391a90dbcd0632818db692700704590264
+size 5969