Training in progress, step 70

adapter_config.json

@@ -6,27 +6,29 @@
   "fan_in_fan_out": null,
   "inference_mode": true,
   "init_lora_weights": true,
+  "layer_replication": null,
   "layers_pattern": null,
   "layers_to_transform": null,
   "loftq_config": {},
-  "lora_alpha": 16,
+  "lora_alpha": 8,
   "lora_dropout": 0.05,
   "megatron_config": null,
   "megatron_core": "megatron.core",
   "modules_to_save": null,
   "peft_type": "LORA",
-  "r": 8,
+  "r": 16,
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
     "up_proj",
+    "q_proj",
+    "k_proj",
     "down_proj",
     "gate_proj",
     "o_proj",
-    "q_proj",
-    "k_proj",
     "v_proj"
   ],
   "task_type": "CAUSAL_LM",
+  "use_dora": false,
   "use_rslora": false
 }

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9314786846103c79129521d1e6078554172c9bbffc72566885f6b2aff8775f2c
-size 80013120
+oid sha256:9cf94ba2d163468bcd1961b5ffd420d151880dd5fda3a4fb93d1152df4a21fa3
+size 80014016

config.json

@@ -18,15 +18,18 @@
   "num_key_value_heads": 32,
   "pretraining_tp": 1,
   "quantization_config": {
-    "bnb_4bit_compute_dtype": "float32",
-    "bnb_4bit_quant_type": "fp4",
-    "bnb_4bit_use_double_quant": false,
+    "_load_in_4bit": true,
+    "_load_in_8bit": false,
+    "bnb_4bit_compute_dtype": "bfloat16",
+    "bnb_4bit_quant_storage": "bfloat16",
+    "bnb_4bit_quant_type": "nf4",
+    "bnb_4bit_use_double_quant": true,
     "llm_int8_enable_fp32_cpu_offload": false,
     "llm_int8_has_fp16_weight": false,
     "llm_int8_skip_modules": null,
     "llm_int8_threshold": 6.0,
-    "load_in_4bit": false,
-    "load_in_8bit": true,
+    "load_in_4bit": true,
+    "load_in_8bit": false,
     "quant_method": "bitsandbytes"
   },
   "rms_norm_eps": 1e-05,
@@ -34,7 +37,7 @@
   "rope_theta": 1000000,
   "tie_word_embeddings": false,
   "torch_dtype": "bfloat16",
-  "transformers_version": "4.37.0",
+  "transformers_version": "4.40.0.dev0",
   "use_cache": false,
   "vocab_size": 32016
 }

tokenizer_config.json

@@ -80,7 +80,6 @@
   "suffix_first": false,
   "suffix_token": "▁<SUF>",
   "tokenizer_class": "CodeLlamaTokenizer",
-  "trust_remote_code": false,
   "unk_token": "<unk>",
   "use_default_system_prompt": false,
   "use_fast": true

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9cf34011f3480c36bc63792fde0cc4fd8f04ef7d4627f8f79baf19897b632d17
-size 4795
+oid sha256:c1a5e882ae94a52ef640dccf5bce9f5f80fa18103f4e60fb6949adaf0f8b1ac4
+size 5816