lora_config.json

{
  "base_model_name_or_path": null,
  "do_qat": false,
  "dtype": "float32",
  "enable_lora_list": null,
  "head_dim": null,
  "lora_alpha": 16,
  "lora_dropout": 0.05,
  "lora_plus_scale": 1.0,
  "lora_use_mixer": false,
  "loraga": false,
  "merge_weights": false,
  "pissa": false,
  "r": 8,
  "rslora": false,
  "scaling": 2.0,
  "target_modules": [
    "language.model.layers.*.self_attn.q_proj.*",
    "language.model.layers.*.self_attn.k_proj.*",
    "language.model.layers.*.self_attn.v_proj.*",
    "language.model.layers.*.self_attn.*o_proj.*",
    "language.model.layers.*.mlp.experts.*.gate_proj.*",
    "language.model.layers.*.mlp.experts.*.up_proj.*",
    "language.model.layers.*.mlp.experts.*.down_proj.*",
    "language.model.layers.*.mlp.gate_proj.*",
    "language.model.layers.*.mlp.up_proj.*",
    "language.model.layers.*.mlp.down_proj.*"
  ],
  "tensor_parallel_degree": 1,
  "trainable_bias": null,
  "trainable_modules": null,
  "use_mora": false,
  "use_quick_lora": false
}