Training in progress, step 50

README.md

@@ -4,8 +4,8 @@ library_name: transformers
 model_name: Meta-Llama-3.1-8B-Instruct-AWQ-INT4_simulated_3-bits_lora
 tags:
 - generated_from_trainer
-- trl
 - sft
+- trl
 licence: license
 ---
 

adapter_config.json

@@ -4,11 +4,18 @@
   "base_model_name_or_path": "hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4",
   "bias": "none",
   "corda_config": null,
-  "eva_config": null,
+  "eva_config": {
+    "adjust_scaling_factors": true,
+    "label_mask_value": -100,
+    "rho": 2.0,
+    "tau": 0.99,
+    "use_label_mask": true,
+    "whiten": false
+  },
   "exclude_modules": null,
   "fan_in_fan_out": false,
   "inference_mode": true,
-  "init_lora_weights": true,
+  "init_lora_weights": "eva",
   "layer_replication": null,
   "layers_pattern": null,
   "layers_to_transform": null,
@@ -25,13 +32,13 @@
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
+    "gate_proj",
+    "down_proj",
     "q_proj",
-    "k_proj",
-    "v_proj",
     "o_proj",
-    "down_proj",
     "up_proj",
-    "gate_proj"
+    "v_proj",
+    "k_proj"
   ],
   "target_parameters": null,
   "task_type": "CAUSAL_LM",

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:89f30b36ab8d7e5c1e5f4027b8ac379dbdd1de5ef263b5b3cbecfff24e2f9fbe
+oid sha256:ba592187ad1db5c29bfd421685286492d45aff68d43da373e12478269271931a
 size 167832240

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4dc31dcae6cf45b6b6d0e1b303a9c79d420a81854e22e52c896c154cab23fd68
+oid sha256:1f5e1440bca299a2a502cfdba638e3636a16a2da6db41726ecc709c0ef6e3388
 size 6353