Training in progress, step 500

adapter_config.json

@@ -10,8 +10,8 @@
   "layers_pattern": null,
   "layers_to_transform": null,
   "loftq_config": {},
-  "lora_alpha": 32,
-  "lora_dropout": 0,
+  "lora_alpha": 128,
+  "lora_dropout": 0.05,
   "megatron_config": null,
   "megatron_core": "megatron.core",
   "modules_to_save": [
@@ -19,17 +19,17 @@
     "embed_tokens"
   ],
   "peft_type": "LORA",
-  "r": 128,
+  "r": 64,
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
+    "v_proj",
+    "q_proj",
+    "up_proj",
     "k_proj",
-    "gate_proj",
     "down_proj",
-    "up_proj",
-    "q_proj",
-    "o_proj",
-    "v_proj"
+    "gate_proj",
+    "o_proj"
   ],
   "task_type": "CAUSAL_LM",
   "use_dora": false,

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:cc9c7876d7156408690c7ccc761aafcbb838cfe1b2b664bcbf07c3f99a0e5965
-size 928031856
+oid sha256:3596e2378a263bb9c76017ad1de3fdb5fc2252ab7b8b772315811250ffcfa5e0
+size 726180672

runs/Sep30_17-48-15_sammie/events.out.tfevents.1727740850.sammie.31562.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3c804c0f96a64899849acb6d539ed3e21b4c1b68271a5f0b19520bd84fe082c0
+size 16281

tokenizer.json

@@ -1,6 +1,11 @@
 {
   "version": "1.0",
-  "truncation": null,
+  "truncation": {
+    "direction": "Right",
+    "max_length": 4096,
+    "strategy": "LongestFirst",
+    "stride": 0
+  },
   "padding": null,
   "added_tokens": [
     {

tokenizer_config.json

@@ -33,9 +33,9 @@
   "clean_up_tokenization_spaces": false,
   "eos_token": "</s>",
   "legacy": false,
-  "model_max_length": 2048,
+  "model_max_length": 4096,
   "pad_token": "<unk>",
-  "padding_side": "left",
+  "padding_side": "right",
   "sp_model_kwargs": {},
   "tokenizer_class": "LlamaTokenizer",
   "unk_token": "<unk>",

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c6129c47eb3d79d123e565041755fd270ca35bbb347ebac7e635faa2854c724d
+oid sha256:cdb3f501f1aa60bff535898f0e98623e86d31015f48e9ec78edeb142fada39c0
 size 5496