adapter_params.json

{"w_bias": false, "lora_layers": "0-32", "lora_rank": 8, "lora_targets": "Q,K,V,O,FFN_DOWN", "lora_alpha": 8, "expert_num": 1, "swi_x": 4, "hydra_moe": true, "p_adapter_layers": "0-32", "p_adapter_size": 16, "p_adapter_hydra": true, "prompt_layers": "0-32", "prompt_len": 10, "expert_weight": false, "max_seq_len": 300, "flash_attention2": false, "bf16": true}