Add library name, pipeline tag and link to Github

README.md

@@ -1,13 +1,1307 @@
 ---
-language:
-- en
 base_model:
 - meta-llama/Llama-3.1-8B-Instruct
+language:
+- en
 tags:
 - peft
 - lora
 - moe
 - moa
+pipeline_tag: text-generation
+library_name: transformers
 ---
 
-arxiv: https://arxiv.org/abs/2506.05928
+arxiv: https://arxiv.org/abs/2506.05928
+
+This repository contains the model weights for the paper [MoA: Heterogeneous Mixture of Adapters for Parameter-Efficient Fine-Tuning of Large Language Models](https://huggingface.co/papers/2506.05928).
+For code and more details, please see the [Github repository](https://github.com/dmis-lab/MoA).
+
+# File information
+
+The repository contains the following file information:
+
+Filename: trainable.json
+Content: {
+  "trainable_params": 24520288,
+  "trainable_params_kv": [
+    [
+      "llama.layers.0.attention.prompt_gate",
+      [
+        1,
+        32,
+        1,
+        1
+      ]
+    ],
+    [
+      "llama.layers.0.attention.lora_Q.lora_A.weight",
+      [
+        8,
+        4096
+      ]
+    ],
+    [
+      "llama.layers.0.attention.lora_Q.lora_B.weight",
+      [
+        4096,
+        8
+      ]
+    ],
+    [
+      "llama.layers.0.attention.lora_K.lora_A.weight",
+      [
+        8,
+        4096
+      ]
+    ],
+    [
+      "llama.layers.0.attention.lora_K.lora_B.weight",
+      [
+        1024,
+        8
+      ]
+    ],
+    [
+      "llama.layers.0.attention.lora_V.lora_A.weight",
+      [
+        8,
+        4096
+      ]
+    ],
+    [
+      "llama.layers.0.attention.lora_V.lora_B.weight",
+      [
+        1024,
+        8
+      ]
+    ],
+    [
+      "llama.layers.0.attention.lora_O.lora_A.weight",
+      [
+        8,
+        4096
+      ]
+    ],
+    [
+      "llama.layers.0.attention.lora_O.lora_B.weight",
+      [
+        4096,
+        8
+      ]
+    ],
+    [
+      "llama.layers.0.attention.prompt.weight",
+      [
+        10,
+        4096
+      ]
+    ],
+    [
+      "llama.layers.0.feed_forward.lora_DOWN.lora_A.weight",
+      [
+        8,
+        14336
+      ]
+    ],
+    [
+      "llama.layers.0.feed_forward.lora_DOWN.lora_B.weight",
+      [
+        4096,
+        8
+      ]
+    ],
+    [
+      "llama.layers.0.p_adapter.down_proj.weight",
+      [
+        16,
+        4096
+      ]
+    ],
+    [
+      "llama.layers.0.p_adapter.down_proj.bias",
+      [
+        16
+      ]
+    ],
+    [
+      "llama.layers.0.p_adapter.up_proj.weight",
+      [
+        4096,
+        16
+      ]
+    ],
+    [
+      "llama.layers.0.p_adapter.up_proj.bias",
+      [
+        4096
+      ]
+    ],
+    [
+      "llama.layers.0.adapter_type_router.w1.weight",
+      [
+        28,
+        4096
+      ]
+    ],
+    [
+      "llama.layers.0.adapter_type_router.w1.bias",
+      [
+        28
+      ]
+    ],
+    [
+      "llama.layers.0.adapter_type_router.w2.weight",
+      [
+        7,
+        28
+      ]
+    ],
+    [
+      "llama.layers.0.adapter_type_router.w2.bias",
+      [
+        7
+      ]
+    ],
+    [
+      "llama.layers.0.adapter_type_router.w3.weight",
+      [
+        28,
+        4096
+      ]
+    ],
+    [
+      "llama.layers.0.adapter_type_router.w3.bias",
+      [
+        28
+      ]
+    ],
+    [
+      "llama.layers.1.attention.prompt_gate",
+      [
+        1,
+        32,
+        1,
+        1
+      ]
+    ],
+    [
+      "llama.layers.1.attention.lora_Q.lora_A.weight",
+      [
+        8,
+        4096
+      ]
+    ],
+    [
+      "llama.layers.1.attention.lora_Q.lora_B.weight",
+      [
+        4096,
+        8
+      ]
+    ],
+    [
+      "llama.layers.1.attention.lora_K.lora_A.weight",
+      [
+        8,
+        4096
+      ]
+    ],
+    [
+      "llama.layers.1.attention.lora_K.lora_B.weight",
+      [
+        1024,
+        8
+      ]
+    ],
+    [
+      "llama.layers.1.attention.lora_V.lora_A.weight",
+      [
+        8,
+        4096
+      ]
+    ],
+    [
+      "llama.layers.1.attention.lora_V.lora_B.weight",
+      [
+        1024,
+        8
+      ]
+    ],
+    [
+      "llama.layers.1.attention.lora_O.lora_A.weight",
+      [
+        8,
+        4096
+      ]
+    ],
+    [
+      "llama.layers.1.attention.lora_O.lora_B.weight",
+      [
+        4096,
+        8
+      ]
+    ],
+    [
+      "llama.layers.1.attention.prompt.weight",
+      [
+        10,
+        4096
+      ]
+    ],
+    [
+      "llama.layers.1.feed_forward.lora_DOWN.lora_A.weight",
+      [
+        8,
+        14336
+      ]
+    ],
+    [
+      "llama.layers.1.feed_forward.lora_DOWN.lora_B.weight",
+      [
+        4096,
+        8
+      ]
+    ],
+    [
+      "llama.layers.1.p_adapter.down_proj.weight",
+      [
+        16,
+        4096
+      ]
+    ],
+    [
+      "llama.layers.1.p_adapter.down_proj.bias",
+      [
+        16
+      ]
+    ],
+    [
+      "llama.layers.1.p_adapter.up_proj.weight",
+      [
+        4096,
+        16
+      ]
+    ],
+    [
+      "llama.layers.1.p_adapter.up_proj.bias",
+      [
+        4096
+      ]
+    ],
+    [
+      "llama.layers.1.adapter_type_router.w1.weight",
+      [
+        28,
+        4096
+      ]
+    ],
+    [
+      "llama.layers.1.adapter_type_router.w1.bias",
+      [
+        28
+      ]
+    ],
+    [
+      "llama.layers.1.adapter_type_router.w2.weight",
+      [
+        7,
+        28
+      ]
+    ],
+    [
+      "llama.layers.1.adapter_type_router.w2.bias",
+      [
+        7
+      ]
+    ],
+    [
+      "llama.layers.1.adapter_type_router.w3.weight",
+      [
+        28,
+        4096
+      ]
+    ],
+    [
+      "llama.layers.1.adapter_type_router.w3.bias",
+      [
+        28
+      ]
+    ],
+    [
+      "llama.layers.2.attention.prompt_gate",
+      [
+        1,
+        32,
+        1,
+        1
+      ]
+    ],
+    [
+      "llama.layers.2.attention.lora_Q.lora_A.weight",
+      [
+        8,
+        4096
+      ]
+    ],
+    [
+      "llama.layers.2.attention.lora_Q.lora_B.weight",
+      [
+        4096,
+        8
+      ]
+    ],
+    [
+      "llama.layers.2.attention.lora_K.lora_A.weight",
+      [
+        8,
+        4096
+      ]
+    ],
+    [
+      "llama.layers.2.attention.lora_K.lora_B.weight",
+      [
+        1024,
+        8
+      ]
+    ],
+    [
+      "llama.layers.2.attention.lora_V.lora_A.weight",
+      [
+        8,
+        4096
+      ]
+    ],
+    [
+      "llama.layers.2.attention.lora_V.lora_B.weight",
+      [
+        1024,
+        8
+      ]
+    ],
+    [
+      "llama.layers.2.attention.lora_O.lora_A.weight",
+      [
+        8,
+        4096
+      ]
+    ],
+    [
+      "llama.layers.2.attention.lora_O.lora_B.weight",
+      [
+        4096,
+        8
+      ]
+    ],
+    [
+      "llama.layers.2.attention.prompt.weight",
+      [
+        10,
+        4096
+      ]
+    ],
+    [
+      "llama.layers.2.feed_forward.lora_DOWN.lora_A.weight",
+      [
+        8,
+        14336
+      ]
+    ],
+    [
+      "llama.layers.2.feed_forward.lora_DOWN.lora_B.weight",
+      [
+        4096,
+        8
+      ]
+    ],
+    [
+      "llama.layers.2.p_adapter.down_proj.weight",
+      [
+        16,
+        4096
+      ]
+    ],
+    [
+      "llama.layers.2.p_adapter.down_proj.bias",
+      [
+        16
+      ]
+    ],
+    [
+      "llama.layers.2.p_adapter.up_proj.weight",
+      [
+        4096,
+        16
+      ]
+    ],
+    [
+      "llama.layers.2.p_adapter.up_proj.bias",
+      [
+        4096
+      ]
+    ],
+    [
+      "llama.layers.2.adapter_type_router.w1.weight",
+      [
+        28,
+        4096
+      ]
+    ],
+    [
+      "llama.layers.2.adapter_type_router.w1.bias",
+      [
+        28
+      ]
+    ],
+    [
+      "llama.layers.2.adapter_type_router.w2.weight",
+      [
+        7,
+        28
+      ]
+    ],
+    [
+      "llama.layers.2.adapter_type_router.w2.bias",
+      [
+        7
+      ]
+    ],
+    [
+      "llama.layers.2.adapter_type_router.w3.weight",
+      [
+        28,
+        4096
+      ]
+    ],
+    [
+      "llama.layers.2.adapter_type_router.w3.bias",
+      [
+        28
+      ]
+    ],
+    [
+      "llama.layers.3.attention.prompt_gate",
+      [
+        1,
+        32,
+        1,
+        1
+      ]
+    ],
+    [
+      "llama.layers.3.attention.lora_Q.lora_A.weight",
+      [
+        8,
+        4096
+      ]
+    ],
+    [
+      "llama.layers.3.attention.lora_Q.lora_B.weight",
+      [
+        4096,
+        8
+      ]
+    ],
+    [
+      "llama.layers.3.attention.lora_K.lora_A.weight",
+      [
+        8,
+        4096
+      ]
+    ],
+    [
+      "llama.layers.3.attention.lora_K.lora_B.weight",
+      [
+        1024,
+        8
+      ]
+    ],
+    [
+      "llama.layers.3.attention.lora_V.lora_A.weight",
+      [
+        8,
+        4096
+      ]
+    ],
+    [
+      "llama.layers.3.attention.lora_V.lora_B.weight",
+      [
+        1024,
+        8
+      ]
+    ],
+    [
+      "llama.layers.3.attention.lora_O.lora_A.weight",
+      [
+        8,
+        4096
+      ]
+    ],
+    [
+      "llama.layers.3.attention.lora_O.lora_B.weight",
+      [
+        4096,
+        8
+      ]
+    ],
+    [
+      "llama.layers.3.attention.prompt.weight",
+      [
+        10,
+        4096
+      ]
+    ],
+    [
+      "llama.layers.3.feed_forward.lora_DOWN.lora_A.weight",
+      [
+        8,
+        14336
+      ]
+    ],
+    [
+      "llama.layers.3.feed_forward.lora_DOWN.lora_B.weight",
+      [
+        4096,
+        8
+      ]
+    ],
+    [
+      "llama.layers.3.p_adapter.down_proj.weight",
+      [
+        16,
+        4096
+      ]
+    ],
+    [
+      "llama.layers.3.p_adapter.down_proj.bias",
+      [
+        16
+      ]
+    ],
+    [
+      "llama.layers.3.p_adapter.up_proj.weight",
+      [
+        4096,
+        16
+      ]
+    ],
+    [
+      "llama.layers.3.p_adapter.up_proj.bias",
+      [
+        4096
+      ]
+    ],
+    [
+      "llama.layers.3.adapter_type_router.w1.weight",
+      [
+        28,
+        4096
+      ]
+    ],
+    [
+      "llama.layers.3.adapter_type_router.w1.bias",
+      [
+        28
+      ]
+    ],
+    [
+      "llama.layers.3.adapter_type_router.w2.weight",
+      [
+        7,
+        28
+      ]
+    ],
+    [
+      "llama.layers.3.adapter_type_router.w2.bias",
+      [
+        7
+      ]
+    ],
+    [
+      "llama.layers.3.adapter_type_router.w3.weight",
+      [
+        28,
+        4096
+      ]
+    ],
+    [
+      "llama.layers.3.adapter_type_router.w3.bias",
+      [
+        28
+      ]
+    ],
+    [
+      "llama.layers.4.attention.prompt_gate",
+      [
+        1,
+        32,
+        1,
+        1
+      ]
+    ],
+    [
+      "llama.layers.4.attention.lora_Q.lora_A.weight",
+      [
+        8,
+        4096
+      ]
+    ],
+    [
+      "llama.layers.4.attention.lora_Q.lora_B.weight",
+      [
+        4096,
+        8
+      ]
+    ],
+    [
+      "llama.layers.4.attention.lora_K.lora_A.weight",
+      [
+        8,
+        4096
+      ]
+    ],
+    [
+      "llama.layers.4.attention.lora_K.lora_B.weight",
+      [
+        1024,
+        8
+      ]
+    ],
+    [
+      "llama.layers.4.attention.lora_V.lora_A.weight",
+      [
+        8,
+        4096
+      ]
+    ],
+    [
+      "llama.layers.4.attention.lora_V.lora_B.weight",
+      [
+        1024,
+        8
+      ]
+    ],
+    [
+      "llama.layers.4.attention.lora_O.lora_A.weight",
+      [
+        8,
+        4096
+      ]
+    ],
+    [
+      "llama.layers.4.attention.lora_O.lora_B.weight",
+      [
+        4096,
+        8
+      ]
+    ],
+    [
+      "llama.layers.4.attention.prompt.weight",
+      [
+        10,
+        4096
+      ]
+    ],
+    [
+      "llama.layers.4.feed_forward.lora_DOWN.lora_A.weight",
+      [
+        8,
+        14336
+      ]
+    ],
+    [
+      "llama.layers.4.feed_forward.lora_DOWN.lora_B.weight",
+      [
+        4096,
+        8
+      ]
+    ],
+    [
+      "llama.layers.4.p_adapter.down_proj.weight",
+      [
+        16,
+        4096
+      ]
+    ],
+    [
+      "llama.layers.4.p_adapter.down_proj.bias",
+      [
+        16
+      ]
+    ],
+    [
+      "llama.layers.4.p_adapter.up_proj.weight",
+      [
+        4096,
+        16
+      ]
+    ],
+    [
+      "llama.layers.4.p_adapter.up_proj.bias",
+      [
+        4096
+      ]
+    ],
+    [
+      "llama.layers.4.adapter_type_router.w1.weight",
+      [
+        28,
+        4096
+      ]
+    ],
+    [
+      "llama.layers.4.adapter_type_router.w1.bias",
+      [
+        28
+      ]
+    ],
+    [
+      "llama.layers.4.adapter_type_router.w2.weight",
+      [
+        7,
+        28
+      ]
+    ],
+    [
+      "llama.layers.4.adapter_type_router.w2.bias",
+      [
+        7
+      ]
+    ],
+    [
+      "llama.layers.4.adapter_type_router.w3.weight",
+      [
+        28,
+        4096
+      ]
+    ],
+    [
+      "llama.layers.4.adapter_type_router.w3.bias",
+      [
+        28
+      ]
+    ],
+    [
+      "llama.layers.5.attention.prompt_gate",
+      [
+        1,
+        32,
+        1,
+        1
+      ]
+    ],
+    [
+      "llama.layers.5.attention.lora_Q.lora_A.weight",
+      [
+        8,
+        4096
+      ]
+    ],
+    [
+      "llama.layers.5.attention.lora_Q.lora_B.weight",
+      [
+        4096,
+        8
+      ]
+    ],
+    [
+      "llama.layers.5.attention.lora_K.lora_A.weight",
+      [
+        8,
+        4096
+      ]
+    ],
+    [
+      "llama.layers.5.attention.lora_K.lora_B.weight",
+      [
+        1024,
+        8
+      ]
+    ],
+    [
+      "llama.layers.5.attention.lora_V.lora_A.weight",
+      [
+        8,
+        4096
+      ]
+    ],
+    [
+      "llama.layers.5.attention.lora_V.lora_B.weight",
+      [
+        1024,
+        8
+      ]
+    ],
+    [
+      "llama.layers.5.attention.lora_O.lora_A.weight",
+      [
+        8,
+        4096
+      ]
+    ],
+    [
+      "llama.layers.5.attention.lora_O.lora_B.weight",
+      [
+        4096,
+        8
+      ]
+    ],
+    [
+      "llama.layers.5.attention.prompt.weight",
+      [
+        10,
+        4096
+      ]
+    ],
+    [
+      "llama.layers.5.feed_forward.lora_DOWN.lora_A.weight",
+      [
+        8,
+        14336
+      ]
+    ],
+    [
+      "llama.layers.5.feed_forward.lora_DOWN.lora_B.weight",
+      [
+        4096,
+        8
+      ]
+    ],
+    [
+      "llama.layers.5.p_adapter.down_proj.weight",
+      [
+        16,
+        4096
+      ]
+    ],
+    [
+      "llama.layers.5.p_adapter.down_proj.bias",
+      [
+        16
+      ]
+    ],
+    [
+      "llama.layers.5.p_adapter.up_proj.weight",
+      [
+        4096,
+        16
+      ]
+    ],
+    [
+      "llama.layers.5.p_adapter.up_proj.bias",
+      [
+        4096
+      ]
+    ],
+    [
+      "llama.layers.5.adapter_type_router.w1.weight",
+      [
+        28,
+        4096
+      ]
+    ],
+    [
+      "llama.layers.5.adapter_type_router.w1.bias",
+      [
+        28
+      ]
+    ],
+    [
+      "llama.layers.5.adapter_type_router.w2.weight",
+      [
+        7,
+        28
+      ]
+    ],
+    [
+      "llama.layers.5.adapter_type_router.w2.bias",
+      [
+        7
+      ]
+    ],
+    [
+      "llama.layers.5.adapter_type_router.w3.weight",
+      [
+        28,
+        4096
+      ]
+    ],
+    [
+      "llama.layers.5.adapter_type_router.w3.bias",
+      [
+        28
+      ]
+    ],
+    [
+      "llama.layers.6.attention.prompt_gate",
+      [
+        1,
+        32,
+        1,
+        1
+      ]
+    ],
+    [
+      "llama.layers.6.attention.lora_Q.lora_A.weight",
+      [
+        8,
+        4096
+      ]
+    ],
+    [
+      "llama.layers.6.attention.lora_Q.lora_B.weight",
+      [
+        4096,
+        8
+      ]
+    ],
+    [
+      "llama.layers.6.attention.lora_K.lora_A.weight",
+      [
+        8,
+        4096
+      ]
+    ],
+    [
+      "llama.layers.6.attention.lora_K.lora_B.weight",
+      [
+        1024,
+        8
+      ]
+    ],
+    [
+      "llama.layers.6.attention.lora_V.lora_A.weight",
+      [
+        8,
+        4096
+      ]
+    ],
+    [
+      "llama.layers.6.attention.lora_V.lora_B.weight",
+      [
+        1024,
+        8
+      ]
+    ],
+    [
+      "llama.layers.6.attention.lora_O.lora_A.weight",
+      [
+        8,
+        4096
+      ]
+    ],
+    [
+      "llama.layers.6.attention.lora_O.lora_B.weight",
+      [
+        4096,
+        8
+      ]
+    ],
+    [
+      "llama.layers.6.attention.prompt.weight",
+      [
+        10,
+        4096
+      ]
+    ],
+    [
+      "llama.layers.6.feed_forward.lora_DOWN.lora_A.weight",
+      [
+        8,
+        14336
+      ]
+    ],
+    [
+      "llama.layers.6.feed_forward.lora_DOWN.lora_B.weight",
+      [
+        4096,
+        8
+      ]
+    ],
+    [
+      "llama.layers.6.p_adapter.down_proj.weight",
+      [
+        16,
+        4096
+      ]
+    ],
+    [
+      "llama.layers.6.p_adapter.down_proj.bias",
+      [
+        16
+      ]
+    ],
+    [
+      "llama.layers.6.p_adapter.up_proj.weight",
+      [
+        4096,
+        16
+      ]
+    ],
+    [
+      "llama.layers.6.p_adapter.up_proj.bias",
+      [
+        4096
+      ]
+    ],
+    [
+      "llama.layers.6.adapter_type_router.w1.weight",
+      [
+        28,
+        4096
+      ]
+    ],
+    [
+      "llama.layers.6.adapter_type_router.w1.bias",
+      [
+        28
+      ]
+    ],
+    [
+      "llama.layers.6.adapter_type_router.w2.weight",
+      [
+        7,
+        28
+      ]
+    ],
+    [
+      "llama.layers.6.adapter_type_router.w2.bias",
+      [
+        7
+      ]
+    ],
+    [
+      "llama.layers.6.adapter_type_router.w3.weight",
+      [
+        28,
+        4096
+      ]
+    ],
+    [
+      "llama.layers.6.adapter_type_router.w3.bias",
+      [
+        28
+      ]
+    ],
+    [
+      "llama.layers.7.attention.prompt_gate",
+      [
+        1,
+        32,
+        1,
+        1
+      ]
+    ],
+    [
+      "llama.layers.7.attention.lora_Q.lora_A.weight",
+      [
+        8,
+        4096
+      ]
+    ],
+    [
+      "llama.layers.7.attention.lora_Q.lora_B.weight",
+      [
+        4096,
+        8
+      ]
+    ],
+    [
+      "llama.layers.7.attention.lora_K.lora_A.weight",
+      [
+        8,
+        4096
+      ]
+    ],
+    [
+      "llama.layers.7.attention.lora_K.lora_B.weight",
+      [
+        1024,
+        8
+      ]
+    ],
+    [
+      "llama.layers.7.attention.lora_V.lora_A.weight",
+      [
+        8,
+        4096
+      ]
+    ],
+    [
+      "llama.layers.7.attention.lora_V.lora_B.weight",
+      [
+        1024,
+        8
+      ]
+    ],
+    [
+      "llama.layers.7.attention.lora_O.lora_A.weight",
+      [
+        8,
+        4096
+      ]
+    ],
+    [
+      "llama.layers.7.attention.lora_O.lora_B.weight",
+      [
+        4096,
+        8
+      ]
+    ],
+    [
+      "llama.layers.7.attention.prompt.weight",
+      [
+        10,
+        4096
+      ]
+    ],
+    [
+      "llama.layers.7.feed_forward.lora_DOWN.lora_A.weight",
+      [
+        8,
+        14336
+      ]
+    ],
+    [
+      "llama.layers.7.feed_forward.lora_DOWN.lora_B.weight",
+      [
+        4096,
+        8
+      ]
+    ],
+    [
+      "llama.layers.7.p_adapter.down_proj.weight",
+      [
+        16,
+        4096
+      ]
+    ],
+    [
+      "llama.layers.7.p_adapter.down_proj.bias",
+      [
+        16
+      ]
+    ],
+    [
+      "llama.layers.7.p_adapter.up_proj.weight",
+      [
+        4096,
+        16
+      ]
+    ],
+    [
+      "llama.layers.7.p_adapter.up_proj.bias",
+      [
+        4096
+      ]
+    ],
+    [
+      "llama.layers.7.adapter_type_router.w1.weight",
+      [
+        28,
+        4096
+      ]
+    ],
+    [
+      "llama.layers.7.adapter_type_router.w1.bias",
+      [
+        28
+      ]
+    ],
+    [
+      "llama.layers.7.adapter_type_router.w2.weight",
+      [
+        7,
+        28
+      ]
+    ],
+    [
+      "llama.layers.7.adapter_type_router.w2.bias",
+      [
+        7
+      ]
+    ],
+    [
+      "llama.layers.7.adapter_type_router.w3.weight",
+      [
+        28,
+        4096
+      ]
+    ],
+    [
+      "llama.layers.7.adapter_type_router.w3.bias",
+      [
+        28
+      ]
+    ],
+    [
+      "llama.layers.8.attention.prompt_gate",
+      [
+        1,
+        32,
+        1,
+        1
+      ]
+    ],
+    [
+      "llama.layers.8.attention.lora_Q.lora_A.weight",
+      [
+        8,
+        4096
+      ]
+    ],
+    [
+      "llama.layers.8.attention.lora_Q.lora_B.weight",
+      [
+        4096,
+        8
+      ]
+    ],
+    [
+      "llama.layers.8.attention.lora_K.lora_A.weight",
+      [
+        8,
+        4096
+      ]
+    ],
+    [
+      "llama.layers.8.attention.lora_K.lora_B.weight",
+      [
+        1024,
+        8
+      ]
+    ],
+    [
+      "llama.layers.8.attention.lora_V.lora_A.weight",
+      [
+        8,
+        4096
+      ]
+    ],
+    [
+      "llama.layers.8.attention.lora_V.lora_B.weight",
+      [
+        1024,
+        8
+      ]
+    ],
+    [
+      "llama.layers.8.attention.lora_O.lora_A.weight",
+      [
+        8,
+        4096
+      ]
+    ],
+    [
+      "llama.layers.8.attention.lora_O.lora_B.weight",
+      [
+        4096,
+        8
+      ]
+    ],
+    [
+      "llama.layers.8.attention.prompt.weight",
+      [
+        10,
+        4096
+      ]
+    ],