{ "dtype": "bfloat16", "architectures": [ "Inkling2LoopForConditionalGeneration" ], "model_type": "inkling2_loop", "eos_token_id": 200006, "text_config": { "model_type": "inkling2_loop_text", "model_max_length": 1048576, "torch_dtype": "bfloat16", "hidden_size": 2304, "num_hidden_layers": 24, "vocab_size": 201024, "num_attention_heads": 24, "num_key_value_heads": 8, "head_dim": 128, "d_rel": 16, "rel_extent": 1024, "q_bias": false, "o_bias": false, "log_scaling_n_floor": 8192, "log_scaling_alpha": 0.1, "rms_norm_eps": 1e-06, "use_value_norm_no_gamma": true, "qk_l2_clip_threshold": 22.627416997969522, "rel_l2_clip_threshold": 8.0, "local_layer_ids": [ 0, 1, 2, 3, 4, 6, 7, 8, 9, 10, 12, 13, 14, 15, 16, 18, 19, 20, 21, 22 ], "sconv_kernel_size": 4, "unpadded_vocab_size": 200058, "final_logit_softcapping": 30.0, "swa_head_dim": 128, "swa_num_attention_heads": 24, "swa_num_key_value_heads": 8, "sliding_window_size": 1024, "n_layers_per_attn_res_block": 6, "attn_res_normalize_values": true, "n_routed_experts": 128, "num_experts_per_tok": 8, "n_shared_experts": 0, "intermediate_size": 1152, "mlp_gate_softcap": 4.0, "mlp_up_softcap": 25.0, "route_scale": 8.0, "use_gate_bias": true, "loop_config": { "core_start": 6, "core_end": 18, "num_passes": 3, "gate_rank": 64, "is_varloop": false } }, "vision_config": { "model_type": "inkling2_vision", "text_hidden_size": 2304, "temporal_patch_size": 2, "native_patch_size": 20, "adapter_merge_size": 2, "projection_hidden_dim": 4096, "projector_bias": false, "hidden_size": 1024, "num_hidden_layers": 23, "num_attention_heads": 16, "mlp_ratio": 4.0, "posemb_grid_size": 32, "layer_norm_eps": 1e-05 }, "mtp_config": { "hidden_size": 2304, "num_hidden_layers": 3, "num_attention_heads": 24, "num_key_value_heads": 8, "head_dim": 128, "intermediate_size": 9216, "max_position_embeddings": 1048576, "torch_dtype": "bfloat16", "q_bias": false, "o_bias": false, "rms_norm_eps": 1e-06, "rope_theta": 1000000.0, "local_layer_ids": [ 0, 2 ], "sliding_window_size": 1024, "block_size": 8, "markov_rank": 256, "markov_head_type": "rnn_hidden", "target_layer_ids": [ 5, 23, 41, 48 ] }, "quantization_config": { "quant_method": "modelopt", "quant_algo": "NVFP4" } }