| { | |
| "adaptive_recurrence_tol": 0.0, | |
| "attention_window": 0, | |
| "attn_res_block_size": 1, | |
| "attn_res_num_blocks": 0, | |
| "bias": false, | |
| "bitnet_act_bits": 8, | |
| "bitnet_hadamard_act": true, | |
| "bitnet_transition_start": 0, | |
| "bitnet_transition_steps": 0, | |
| "block_size": 512, | |
| "blt_cross_k": 2, | |
| "blt_decoder_layers": 2, | |
| "blt_encoder_layers": 2, | |
| "blt_entropy_threshold": 1.0, | |
| "blt_local_dim": 128, | |
| "blt_local_heads": 4, | |
| "blt_local_window": 128, | |
| "blt_max_patch": 8, | |
| "blt_max_patches": 512, | |
| "blt_ngram_sizes": [ | |
| 2, | |
| 3, | |
| 4 | |
| ], | |
| "blt_ngram_vocab": 50021, | |
| "blt_patcher_dim": 96, | |
| "blt_patcher_layers": 2, | |
| "blt_pool_levels": 1, | |
| "blt_superpatch_size": 4, | |
| "bos_token_id": 1, | |
| "deep_supervision_weight": 0.3, | |
| "dspark_block_size": 4, | |
| "dspark_ce_alpha": 0.1, | |
| "dspark_conf_alpha": 1.0, | |
| "dspark_confidence_with_markov": true, | |
| "dspark_loss_decay_gamma": 4.0, | |
| "dspark_loss_weight": 1.0, | |
| "dspark_markov_head": "vanilla", | |
| "dspark_markov_rank": 64, | |
| "dspark_n_layers": 2, | |
| "dspark_num_anchors": 4, | |
| "dspark_tv_alpha": 0.9, | |
| "dspark_use_confidence_head": true, | |
| "effective_expected_depth": 12, | |
| "eos_token_id": 2, | |
| "head_dim": 64, | |
| "init_values": { | |
| "embed_scale": 16.0, | |
| "embedding": 0.03952847075210474, | |
| "out_proj": 0.008068715304598786, | |
| "std": 0.03952847075210474 | |
| }, | |
| "injection_type": "lti", | |
| "intermediate_size": 768, | |
| "kv_lora_rank": 512, | |
| "max_step_embeddings": 64, | |
| "mean_backprop_depth": 3, | |
| "mean_recurrence": 4, | |
| "memory_chunk_size": 32, | |
| "memory_dim": 128, | |
| "mhc_h_res_proj": "sinkhorn", | |
| "mhc_num_streams": 4, | |
| "mhc_residual_alpha": 0.01, | |
| "mhc_residual_identity_mix": false, | |
| "mhc_sinkhorn_iters": 10, | |
| "mhc_sinkhorn_tau": 0.05, | |
| "model_type": "anglofable", | |
| "moe_bias_update_speed": 0.001, | |
| "moe_intermediate_size": 1024, | |
| "moe_top_k": 2, | |
| "mor_capacity": 0.5, | |
| "mor_min_steps": 1, | |
| "mor_ponder_weight": 0.01, | |
| "n_embd": 256, | |
| "n_heads": 4, | |
| "n_kv_heads": 4, | |
| "n_layers": 8, | |
| "n_layers_in_coda": 2, | |
| "n_layers_in_prelude": 2, | |
| "n_layers_in_recurrent_block": 2, | |
| "n_routed_experts": 8, | |
| "n_shared_experts": 2, | |
| "norm_eps": 1e-06, | |
| "num_key_value_heads": 4, | |
| "pad_token_id": 0, | |
| "q_lora_rank": 1536, | |
| "qk_bias": true, | |
| "rope_base": 50000.0, | |
| "rope_head_dim": 64, | |
| "state_init": "like-init", | |
| "stochastic_depth_rate": 0.1, | |
| "tie_embeddings": true, | |
| "tie_word_embeddings": true, | |
| "transformers_version": "5.6.2", | |
| "turboquant_kv_bits": 4, | |
| "turboquant_qat": false, | |
| "use_attn_res": true, | |
| "use_bitnet": false, | |
| "use_deep_supervision": true, | |
| "use_dspark": false, | |
| "use_mhc": true, | |
| "use_mla": false, | |
| "use_moe": false, | |
| "use_moe_bias_balance": true, | |
| "use_mor": false, | |
| "use_neural_memory": true, | |
| "use_recurrence_norm": true, | |
| "use_step_embedding": true, | |
| "use_stochastic_depth": false, | |
| "use_turboquant": true, | |
| "use_world_model": true, | |
| "vocab_size": 32000, | |
| "wm_ema_decay": 0.998, | |
| "wm_latent_dim": 256, | |
| "wm_prediction_loss_weight": 0.1, | |
| "wm_predictor_heads": 4, | |
| "wm_predictor_layers": 1, | |
| "wm_reg_type": "sigreg", | |
| "wm_sigreg_projections": 64, | |
| "wm_use_adaln": true, | |
| "wm_use_ema_target": true, | |
| "wm_variance_reg_weight": 0.01 | |
| } | |