config.json

{
  "add_bias_linear": false,
  "add_qkv_bias": false,
  "apply_residual_connection_post_layernorm": false,
  "architectures": [
    "NCPOlmo3ForCausalLM"
  ],
  "attention_dropout": 0.0,
  "auto_map": {
    "AutoConfig": "configuration_ncp_olmo3.NCPOlmo3Config",
    "AutoModelForCausalLM": "modeling_ncp_olmo3.NCPOlmo3ForCausalLM"
  },
  "checkpoint_step": 700000,
  "concept_chunk_size": 4,
  "concept_shift_feature": true,
  "conceptlm_backbone": "olmo3",
  "conceptlm_chunk_dualpath_alpha_init": 0.5,
  "conceptlm_chunk_merge_method": "meanpooling",
  "conceptlm_chunk_size": 4,
  "conceptlm_decoder_layers": 16,
  "conceptlm_enable_chunk_dualpath_smoothing": false,
  "conceptlm_encoder_layers": 16,
  "conceptlm_fusion_alpha_init": 1.0,
  "conceptlm_fusion_norm_alpha_init": 0.1,
  "conceptlm_hlm_attention_mode": "backbone_window",
  "conceptlm_hlm_ffn_hidden_size": null,
  "conceptlm_hlm_loss_weight": 1.0,
  "conceptlm_layer_norm_option": "normed_add",
  "conceptlm_shift_feature": true,
  "conceptlm_special_layers": 8,
  "conceptlm_v21_compile_dd_routes": true,
  "conceptlm_v21_compile_residual_flow_routes": true,
  "conceptlm_v21_concept_read_encoder_first_n": -1,
  "conceptlm_v21_dd_concept_self_dd": true,
  "conceptlm_v21_dd_concept_self_dd_every_n_layers": 1,
  "conceptlm_v21_dd_concept_self_dd_hidden_size": 0,
  "conceptlm_v21_dd_concept_self_dd_use_layernorm": false,
  "conceptlm_v21_dd_encoder_self_dd": true,
  "conceptlm_v21_dd_encoder_self_dd_every_n_layers": 1,
  "conceptlm_v21_dd_encoder_self_dd_hidden_size": 0,
  "conceptlm_v21_dd_encoder_self_dd_use_layernorm": false,
  "conceptlm_v21_dd_self_dd_mode": "dd",
  "conceptlm_v21_dd_two_route_add": true,
  "conceptlm_v21_dd_two_route_add_beta_init": 0.05,
  "conceptlm_v21_dd_two_route_add_concept_hidden_size": 0,
  "conceptlm_v21_dd_two_route_add_concept_route_first_n": -1,
  "conceptlm_v21_dd_two_route_add_concept_source": "final",
  "conceptlm_v21_dd_two_route_add_concept_use_layernorm": true,
  "conceptlm_v21_dd_two_route_add_decoder_hidden_size": 0,
  "conceptlm_v21_dd_two_route_add_decoder_use_layernorm": false,
  "conceptlm_v21_dd_two_route_add_decoder_use_softmax": true,
  "conceptlm_v21_dd_two_route_add_disable_decoder_dd": false,
  "conceptlm_v21_dd_two_route_add_enable_final_concept_route": true,
  "conceptlm_v21_dd_two_route_add_enable_raw_concept_route": false,
  "conceptlm_v21_dd_two_route_add_every_n_layers": 1,
  "conceptlm_v21_dd_two_route_add_use_softmax": false,
  "conceptlm_v21_decoder_read_encoder_first_n": -1,
  "conceptlm_v21_enable_concept_read_encoder": true,
  "conceptlm_v21_enable_decoder_read_concept": true,
  "conceptlm_v21_enable_decoder_read_encoder": true,
  "conceptlm_v21_enable_full_residual_flow": false,
  "conceptlm_v21_final_read_concept_gate": true,
  "conceptlm_v21_final_read_concept_gate_init_final": 0.5,
  "conceptlm_v21_final_read_concept_gate_reg_weight": 0.0,
  "conceptlm_v21_final_read_concept_gate_target_final": 0.5,
  "conceptlm_v21_residual_flow_beta_init": 0.02,
  "conceptlm_v21_residual_flow_route_hidden_size": 0,
  "conceptlm_v21_residual_flow_route_use_softmax": true,
  "conceptlm_v21_residual_flow_shared_source_norm": true,
  "conceptlm_v21_residual_flow_source_use_layernorm": true,
  "conceptlm_v22_vq_codebook_size": 128,
  "conceptlm_v22_vq_commitment_cost": 0.25,
  "conceptlm_v22_vq_detach_hlm_target": true,
  "conceptlm_v22_vq_highlevel_lr_mult": 1.0,
  "conceptlm_v22_vq_hlm_loss_type": "mse",
  "conceptlm_v22_vq_lr_mult": 1.0,
  "conceptlm_v22_vq_merge_mode": "raw_logits",
  "conceptlm_v22_vq_num_codebooks": 32,
  "conceptlm_vq_loss_weight": 1.0,
  "custom_model_loader_required": false,
  "ffn_hidden_size": 11008,
  "fp32_residual_connection": false,
  "gated_linear_unit": true,
  "hidden_dropout": 0.0,
  "hidden_size": 4096,
  "kv_channels": 128,
  "layernorm_epsilon": 1e-06,
  "layernorm_zero_centered_gamma": false,
  "max_position_embeddings": 8192,
  "max_sequence_length": 8192,
  "model_type": "ncp_olmo3",
  "normalization": "RMSNorm",
  "num_attention_heads": 32,
  "num_hidden_layers": 40,
  "num_layers": 32,
  "num_query_groups": 32,
  "params_dtype": "bfloat16",
  "position_embedding_type": "rope",
  "qk_layernorm": true,
  "rotary_base": 500000,
  "rotary_interleaved": false,
  "rotary_percent": 1.0,
  "standalone_hf_backend": true,
  "standalone_hf_backend_status": "parity_candidate",
  "torch_dtype": "bfloat16",
  "transformers_version": "5.8.0",
  "untie_embeddings_and_output_weights": true,
  "use_cache": true,
  "vocab_size": 100278,
  "weight_key_format": "native_megatron_state_dict",
  "window_attn_skip_freq": 4,
  "window_size": [
    4096,
    0
  ]
}