workmate-pretrain

README.md

@@ -1,9 +1,9 @@
 ---
+library_name: peft
 license: gemma
 base_model: google/gemma-2b
 tags:
 - generated_from_trainer
-library_name: peft
 model-index:
 - name: lora_optput
   results: []
@@ -34,17 +34,17 @@ More information needed
 
 The following hyperparameters were used during training:
 - learning_rate: 0.0002
-- train_batch_size: 4
-- eval_batch_size: 8
+- train_batch_size: 8
+- eval_batch_size: 16
 - seed: 42
-- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- optimizer: Use adamw_torch with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
 - lr_scheduler_type: linear
 - num_epochs: 3
 
 ### Framework versions
 
-- PEFT 0.10.0
-- Transformers 4.40.0
-- Pytorch 2.2.2
-- Datasets 2.18.0
-- Tokenizers 0.19.1
+- PEFT 0.14.0
+- Transformers 4.47.0
+- Pytorch 2.5.1+cu121
+- Datasets 3.3.1
+- Tokenizers 0.21.0

adapter_config.json

@@ -3,6 +3,8 @@
   "auto_mapping": null,
   "base_model_name_or_path": "google/gemma-2b",
   "bias": "none",
+  "eva_config": null,
+  "exclude_modules": null,
   "fan_in_fan_out": false,
   "inference_mode": true,
   "init_lora_weights": true,
@@ -11,6 +13,7 @@
   "layers_to_transform": null,
   "loftq_config": {},
   "lora_alpha": 16,
+  "lora_bias": false,
   "lora_dropout": 0.05,
   "megatron_config": null,
   "megatron_core": "megatron.core",
@@ -23,7 +26,7 @@
     "q_proj",
     "v_proj"
   ],
-  "task_type": "CASUSL_LM",
+  "task_type": "CAUSAL_LM",
   "use_dora": false,
   "use_rslora": false
 }

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:be175e019b4eb86e0bb9856d7c95e8357179df757175e83de0d5b2a447e5b32e
+oid sha256:25745ee49d1a33572183beb853dc7c134155c206df28c5c95383830f9b1365e3
 size 3695848

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2b3892a11c5d0a47eadf12115d7f8d66a43b3794edccc4dd9b9af0428e8c66b1
-size 4920
+oid sha256:5d74ebef0249a792f06f792ff1851821ca3aea4f39fe455bbed250acae5f9978
+size 5304