Training in progress, step 756

adapter_config.json

@@ -13,7 +13,7 @@
   "layers_pattern": null,
   "layers_to_transform": null,
   "loftq_config": {},
-  "lora_alpha": 32,
+  "lora_alpha": 64,
   "lora_bias": false,
   "lora_dropout": 0.0,
   "megatron_config": null,
@@ -21,14 +21,14 @@
   "modules_to_save": null,
   "peft_type": "LORA",
   "qalora_group_size": 16,
-  "r": 16,
+  "r": 32,
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
-    "q_proj",
     "k_proj",
     "v_proj",
-    "o_proj"
+    "o_proj",
+    "q_proj"
   ],
   "target_parameters": null,
   "task_type": "CAUSAL_LM",

special_tokens_map.json

@@ -12,5 +12,6 @@
     "normalized": false,
     "rstrip": false,
     "single_word": false
-  }
+  },
+  "pad_token": "<|eot_id|>"
 }

tokenizer_config.json

@@ -2058,5 +2058,6 @@
     "attention_mask"
   ],
   "model_max_length": 131072,
+  "pad_token": "<|eot_id|>",
   "tokenizer_class": "PreTrainedTokenizerFast"
 }

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:03b1942d9ae44a16ea382fa95ff366a088f3a37a04e1cb71abdcdc73d1c465b4
+oid sha256:8ed998849d023f4444ee38d4d825e257bb05b3306a34e0b8999132e9ee2c38b9
 size 5777