Upload folder using huggingface_hub

README.md

@@ -1,9 +1,9 @@
 ---
 base_model:
-- mllm-dev/gpt2_f_experiment_2_drug_data_new_run
+- mllm-dev/gpt2_f_experiment_0_drug_data_new_run
 - mllm-dev/gpt2_f_experiment_4_drug_data_new_run
+- mllm-dev/gpt2_f_experiment_2_drug_data_new_run
 - mllm-dev/gpt2_f_experiment_1_drug_data_new_run
-- mllm-dev/gpt2_f_experiment_0_drug_data_new_run
 - mllm-dev/gpt2_f_experiment_3_drug_data_new_run
 library_name: transformers
 tags:
@@ -11,22 +11,21 @@ tags:
 - merge
 
 ---
-# tam_test_merge_out_drug_data_linear_test_new_run
+# tam_test_merge_out_drug_data_dare_linear_test_new_run
 
 This is a merge of pre-trained language models created using [mergekit](https://github.com/cg123/mergekit).
 
 ## Merge Details
 ### Merge Method
 
-This model was merged using the [linear](https://arxiv.org/abs/2203.05482) merge method.
+This model was merged using the linear [DARE](https://arxiv.org/abs/2311.03099) merge method using [mllm-dev/gpt2_f_experiment_0_drug_data_new_run](https://huggingface.co/mllm-dev/gpt2_f_experiment_0_drug_data_new_run) as a base.
 
 ### Models Merged
 
 The following models were included in the merge:
-* [mllm-dev/gpt2_f_experiment_2_drug_data_new_run](https://huggingface.co/mllm-dev/gpt2_f_experiment_2_drug_data_new_run)
 * [mllm-dev/gpt2_f_experiment_4_drug_data_new_run](https://huggingface.co/mllm-dev/gpt2_f_experiment_4_drug_data_new_run)
+* [mllm-dev/gpt2_f_experiment_2_drug_data_new_run](https://huggingface.co/mllm-dev/gpt2_f_experiment_2_drug_data_new_run)
 * [mllm-dev/gpt2_f_experiment_1_drug_data_new_run](https://huggingface.co/mllm-dev/gpt2_f_experiment_1_drug_data_new_run)
-* [mllm-dev/gpt2_f_experiment_0_drug_data_new_run](https://huggingface.co/mllm-dev/gpt2_f_experiment_0_drug_data_new_run)
 * [mllm-dev/gpt2_f_experiment_3_drug_data_new_run](https://huggingface.co/mllm-dev/gpt2_f_experiment_3_drug_data_new_run)
 
 ### Configuration
@@ -34,28 +33,31 @@ The following models were included in the merge:
 The following YAML configuration was used to produce this model:
 
 ```yaml
+base_model: mllm-dev/gpt2_f_experiment_0_drug_data_new_run
 dtype: float16
-merge_method: linear
+merge_method: dare_linear
+parameters:
+  normalize: 1.0
 slices:
 - sources:
   - layer_range: [0, 12]
     model: mllm-dev/gpt2_f_experiment_0_drug_data_new_run
     parameters:
-      weight: 1.0
+      weight: 0.2
   - layer_range: [0, 12]
     model: mllm-dev/gpt2_f_experiment_1_drug_data_new_run
     parameters:
-      weight: 1.0
+      weight: 0.2
   - layer_range: [0, 12]
     model: mllm-dev/gpt2_f_experiment_2_drug_data_new_run
     parameters:
-      weight: 1.0
+      weight: 0.2
   - layer_range: [0, 12]
     model: mllm-dev/gpt2_f_experiment_3_drug_data_new_run
     parameters:
-      weight: 1.0
+      weight: 0.2
   - layer_range: [0, 12]
     model: mllm-dev/gpt2_f_experiment_4_drug_data_new_run
     parameters:
-      weight: 1.0
+      weight: 0.2
 ```

config.json

@@ -1,5 +1,5 @@
 {
-  "_name_or_path": "mllm-dev/gpt2_f_experiment_2_drug_data_new_run",
+  "_name_or_path": "mllm-dev/gpt2_f_experiment_0_drug_data_new_run",
   "activation_function": "gelu_new",
   "architectures": [
     "GPT2ForSequenceClassification"

mergekit_config.yml

@@ -1,24 +1,27 @@
+base_model: mllm-dev/gpt2_f_experiment_0_drug_data_new_run
 dtype: float16
-merge_method: linear
+merge_method: dare_linear
+parameters:
+  normalize: 1.0
 slices:
 - sources:
   - layer_range: [0, 12]
     model: mllm-dev/gpt2_f_experiment_0_drug_data_new_run
     parameters:
-      weight: 1.0
+      weight: 0.2
   - layer_range: [0, 12]
     model: mllm-dev/gpt2_f_experiment_1_drug_data_new_run
     parameters:
-      weight: 1.0
+      weight: 0.2
   - layer_range: [0, 12]
     model: mllm-dev/gpt2_f_experiment_2_drug_data_new_run
     parameters:
-      weight: 1.0
+      weight: 0.2
   - layer_range: [0, 12]
     model: mllm-dev/gpt2_f_experiment_3_drug_data_new_run
     parameters:
-      weight: 1.0
+      weight: 0.2
   - layer_range: [0, 12]
     model: mllm-dev/gpt2_f_experiment_4_drug_data_new_run
     parameters:
-      weight: 1.0
+      weight: 0.2

model-00001-of-00001.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:45d2c76b0cd7575720adb18f45b7b7d18ba048a4e78c89a7382364eb5c92c58b
+oid sha256:66898369bbead1e332e534f5a64b4d3f23601a74d6384ee152e68560e680fc91
 size 248909944