| { |
| "add_bias_linear": false, |
| "add_qkv_bias": false, |
| "apply_residual_connection_post_layernorm": false, |
| "architectures": [ |
| "NCPOlmo3ForCausalLM" |
| ], |
| "attention_dropout": 0.0, |
| "auto_map": { |
| "AutoConfig": "configuration_ncp_olmo3.NCPOlmo3Config", |
| "AutoModelForCausalLM": "modeling_ncp_olmo3.NCPOlmo3ForCausalLM" |
| }, |
| "checkpoint_step": 1200000, |
| "concept_chunk_size": 4, |
| "concept_shift_feature": true, |
| "conceptlm_backbone": "olmo3", |
| "conceptlm_chunk_dualpath_alpha_init": 0.5, |
| "conceptlm_chunk_merge_method": "meanpooling", |
| "conceptlm_chunk_size": 4, |
| "conceptlm_decoder_layers": 16, |
| "conceptlm_enable_chunk_dualpath_smoothing": false, |
| "conceptlm_encoder_layers": 16, |
| "conceptlm_fusion_alpha_init": 1.0, |
| "conceptlm_fusion_norm_alpha_init": 0.1, |
| "conceptlm_hlm_attention_mode": "backbone_window", |
| "conceptlm_hlm_ffn_hidden_size": null, |
| "conceptlm_hlm_loss_weight": 1.0, |
| "conceptlm_layer_norm_option": "normed_add", |
| "conceptlm_shift_feature": true, |
| "conceptlm_special_layers": 8, |
| "conceptlm_v21_compile_dd_routes": true, |
| "conceptlm_v21_compile_residual_flow_routes": true, |
| "conceptlm_v21_concept_read_encoder_first_n": -1, |
| "conceptlm_v21_dd_concept_self_dd": true, |
| "conceptlm_v21_dd_concept_self_dd_every_n_layers": 1, |
| "conceptlm_v21_dd_concept_self_dd_hidden_size": 0, |
| "conceptlm_v21_dd_concept_self_dd_use_layernorm": false, |
| "conceptlm_v21_dd_encoder_self_dd": true, |
| "conceptlm_v21_dd_encoder_self_dd_every_n_layers": 1, |
| "conceptlm_v21_dd_encoder_self_dd_hidden_size": 0, |
| "conceptlm_v21_dd_encoder_self_dd_use_layernorm": false, |
| "conceptlm_v21_dd_self_dd_mode": "dd", |
| "conceptlm_v21_dd_two_route_add": true, |
| "conceptlm_v21_dd_two_route_add_beta_init": 0.05, |
| "conceptlm_v21_dd_two_route_add_concept_hidden_size": 0, |
| "conceptlm_v21_dd_two_route_add_concept_route_first_n": -1, |
| "conceptlm_v21_dd_two_route_add_concept_source": "final", |
| "conceptlm_v21_dd_two_route_add_concept_use_layernorm": true, |
| "conceptlm_v21_dd_two_route_add_decoder_hidden_size": 0, |
| "conceptlm_v21_dd_two_route_add_decoder_use_layernorm": false, |
| "conceptlm_v21_dd_two_route_add_decoder_use_softmax": true, |
| "conceptlm_v21_dd_two_route_add_disable_decoder_dd": false, |
| "conceptlm_v21_dd_two_route_add_enable_final_concept_route": true, |
| "conceptlm_v21_dd_two_route_add_enable_raw_concept_route": false, |
| "conceptlm_v21_dd_two_route_add_every_n_layers": 1, |
| "conceptlm_v21_dd_two_route_add_use_softmax": false, |
| "conceptlm_v21_decoder_read_encoder_first_n": -1, |
| "conceptlm_v21_enable_concept_read_encoder": true, |
| "conceptlm_v21_enable_decoder_read_concept": true, |
| "conceptlm_v21_enable_decoder_read_encoder": true, |
| "conceptlm_v21_enable_full_residual_flow": false, |
| "conceptlm_v21_final_read_concept_gate": true, |
| "conceptlm_v21_final_read_concept_gate_init_final": 0.5, |
| "conceptlm_v21_final_read_concept_gate_reg_weight": 0.0, |
| "conceptlm_v21_final_read_concept_gate_target_final": 0.5, |
| "conceptlm_v21_residual_flow_beta_init": 0.02, |
| "conceptlm_v21_residual_flow_route_hidden_size": 0, |
| "conceptlm_v21_residual_flow_route_use_softmax": true, |
| "conceptlm_v21_residual_flow_shared_source_norm": true, |
| "conceptlm_v21_residual_flow_source_use_layernorm": true, |
| "conceptlm_v22_vq_codebook_size": 128, |
| "conceptlm_v22_vq_commitment_cost": 0.25, |
| "conceptlm_v22_vq_detach_hlm_target": true, |
| "conceptlm_v22_vq_highlevel_lr_mult": 1.0, |
| "conceptlm_v22_vq_hlm_loss_type": "mse", |
| "conceptlm_v22_vq_lr_mult": 1.0, |
| "conceptlm_v22_vq_merge_mode": "raw_logits", |
| "conceptlm_v22_vq_num_codebooks": 32, |
| "conceptlm_vq_loss_weight": 1.0, |
| "custom_model_loader_required": false, |
| "ffn_hidden_size": 11008, |
| "fp32_residual_connection": false, |
| "gated_linear_unit": true, |
| "hidden_dropout": 0.0, |
| "hidden_size": 4096, |
| "kv_channels": 128, |
| "layernorm_epsilon": 1e-06, |
| "layernorm_zero_centered_gamma": false, |
| "max_position_embeddings": 8192, |
| "max_sequence_length": 8192, |
| "model_type": "ncp_olmo3", |
| "normalization": "RMSNorm", |
| "num_attention_heads": 32, |
| "num_hidden_layers": 40, |
| "num_layers": 32, |
| "num_query_groups": 32, |
| "params_dtype": "bfloat16", |
| "position_embedding_type": "rope", |
| "qk_layernorm": true, |
| "rotary_base": 500000, |
| "rotary_interleaved": false, |
| "rotary_percent": 1.0, |
| "standalone_hf_backend": true, |
| "standalone_hf_backend_status": "parity_candidate", |
| "torch_dtype": "bfloat16", |
| "transformers_version": "5.8.0", |
| "untie_embeddings_and_output_weights": true, |
| "use_cache": true, |
| "vocab_size": 100278, |
| "weight_key_format": "native_megatron_state_dict", |
| "window_attn_skip_freq": 4, |
| "window_size": [ |
| 4096, |
| 0 |
| ] |
| } |
|
|
