Update config.json

config.json

@@ -9,7 +9,6 @@
     "AutoModelForCausalLM": "modeling_motif.MotifForCausalLM"
   },
   "batch_num": null,
-  "bfloat16": true,
   "bos_token_id": 219396,
   "continual_training": false,
   "decoder_split_layers": [],
@@ -19,24 +18,17 @@
   "dim_model_base_lmh": 1,
   "dim_model_base_logits": 2048,
   "dim_model_base_lr": 256,
-  "down_proj_alpha": 0.15625,
-  "embed_tokens_alpha": null,
   "encoder_split_layers": [],
   "eos_token_id": 219395,
   "first_expansion": false,
   "fused_rope": false,
-  "gate_up_proj_alpha": 0.15625,
   "hidden_act": "poly_norm",
   "hidden_act_moe": null,
   "hidden_size": 2048,
   "hidden_states_shrink": 0.17677669529663687,
   "init_scale_o": 1,
   "initializer_range": 2e-05,
-  "input_layernorm_alpha": null,
   "intermediate_size": 8192,
-  "k_proj_alpha": 0.15625,
-  "lm_head_alpha": null,
-  "load_pretrained": "",
   "loss_reduction": "mean",
   "max_position_embeddings": 16384,
   "max_window_layers": 28,
@@ -49,15 +41,10 @@
   "multi_token_heads": null,
   "n_group": null,
   "n_routed_experts": null,
-  "norm_alpha": null,
   "norm_topk_prob": null,
   "num_attention_heads": 16,
   "num_hidden_layers": 32,
   "num_key_value_heads": 16,
-  "num_stages": 3,
-  "o_proj_alpha": 0.15625,
-  "post_attention_layernorm_alpha": null,
-  "q_proj_alpha": 0.15625,
   "rms_norm_eps": 1e-06,
   "rope_scaling": null,
   "rope_theta": 500000.0,
@@ -66,25 +53,20 @@
   "scoring_func": null,
   "seq_aux": null,
   "sliding_window": null,
-  "tensor_parallel": false,
   "tie_word_embeddings": true,
   "topk_group": null,
   "topk_method": null,
-  "torch_dtype": "float32",
+  "torch_dtype": "bfloat16",
   "transformers_version": "4.46.3",
   "use_advanced_parallelization": false,
   "use_bias": false,
   "use_cache": true,
-  "use_emb_alpha": false,
   "use_fused_mlp": null,
   "use_moreh_attention": false,
   "use_moreh_moe": false,
   "use_mrope": false,
-  "use_norm_alpha": false,
-  "use_pipeline": false,
   "use_qk_norm": false,
   "use_sliding_window": false,
-  "v_proj_alpha": 0.15625,
   "vocab_size": 219520,
   "wesar_weights": false
 }