| { |
| "affine_momentum": 0.9, |
| "architectures": [ |
| "NeoLLMForCausalLM" |
| ], |
| "attention_bias": false, |
| "attention_dropout": 0.1, |
| "attn_res_num_blocks": 4, |
| "auto_map": { |
| "AutoConfig": "configuration_neollm.NeoLLMConfig", |
| "AutoModel": "modeling_neollm.NeoLLMModel", |
| "AutoModelForCausalLM": "modeling_neollm.NeoLLMForCausalLM" |
| }, |
| "bos_token_id": 1, |
| "cce_loss_impl": "cce_kahan_full_c", |
| "dagformer": { |
| "input_source": "tokens", |
| "local_dim": 128, |
| "predictor_dim": 256, |
| "predictor_heads": 4, |
| "predictor_layers": 2 |
| }, |
| "directional_routing_k": 4, |
| "directional_routing_temp": 3.0, |
| "dropout_rate": 0.1, |
| "dtype": "bfloat16", |
| "dynamics_metrics_enabled": true, |
| "dynamics_sample_tokens": 64, |
| "eos_token_id": 7, |
| "fan_ratio": 0.125, |
| "fan_ratio_ffn": 0.0625, |
| "generator_d_seed": 128, |
| "generator_k": 3, |
| "generator_krank": 64, |
| "generator_num_knots": 16, |
| "generator_num_modes": 8, |
| "generator_spline_degree": 2, |
| "head_dim": 64, |
| "hidden_act": "xielu", |
| "hidden_size": 512, |
| "hola_chunk_size": 256, |
| "hola_logit_bias_init": -2.0, |
| "hola_memory_size": 64, |
| "hola_rms_gamma_init": 1.0, |
| "hola_score_eps": 1e-06, |
| "iha_global_layers_use_iha": false, |
| "iha_local_global_pattern": "LLLLG", |
| "iha_num_pseudo_heads": 2, |
| "iha_sliding_window": null, |
| "initializer_range": 0.02, |
| "intermediate_size": 1536, |
| "jtok_aux_loss_weight": 0.0001, |
| "jtok_kernel_backend": "triton", |
| "jtok_norm_eps": 1e-06, |
| "jtok_num_experts": 5, |
| "jtok_num_modes": 4, |
| "jtok_top_k": 2, |
| "laurel_lr_rank": 32, |
| "liger_loss_accum_dtype": "float32", |
| "lucid_attention_eps": 1e-06, |
| "max_position_embeddings": 512, |
| "mea_component_key_value_heads": 4, |
| "mea_groupnorm_eps": 1e-06, |
| "meap_exclude_last": true, |
| "meap_implementation": "triton", |
| "meap_mask_ratio": 0.15, |
| "meap_mask_token_id": 16, |
| "meap_seed": 42, |
| "mile_loss_gamma": 1.0, |
| "model_type": "neollm", |
| "momentum_gamma": 0.1, |
| "mu_loss_lambda": 0.0001, |
| "nextlat_dynamics_proj_factor": 1.0, |
| "nextlat_horizon": 1, |
| "nextlat_kl_weight": 1.0, |
| "nextlat_mse_weight": 1.0, |
| "nitp_loss_weight": 1.0, |
| "nitp_projector_intermediate_size": 2048, |
| "nitp_target_layer": 2, |
| "nitp_temporal_apply_loss": true, |
| "nitp_temporal_dynamics_weight": 1.0, |
| "nitp_temporal_horizon": 4, |
| "nitp_temporal_identification_weight": 0.1, |
| "nitp_temporal_intermediate_size": 1024, |
| "nitp_temporal_similarity_temperature": 0.1, |
| "nitp_temporal_target_temperature": 0.5, |
| "ntp_loss_backend": "cce", |
| "num_attention_heads": 8, |
| "num_hidden_layers": 12, |
| "num_key_value_heads": 4, |
| "num_mem_heads": 4, |
| "pad_token_id": 0, |
| "partial_rotary_factor": 0.25, |
| "polynorm_exclusive": false, |
| "repo_d_p": 64, |
| "repo_goat_num_frequencies": 3, |
| "repo_goat_sink_decay": 4.0, |
| "repo_grape_circular_prior_strength": 0.01, |
| "repo_start_layer": 4, |
| "rms_norm_eps": 1e-06, |
| "rope_parameters": { |
| "partial_rotary_factor": 0.25, |
| "rope_theta": 10000.0, |
| "rope_type": "default" |
| }, |
| "rope_theta": 10000.0, |
| "siamese_attn_x_scale_init": 1.0, |
| "siamese_depth_scaling": true, |
| "siamese_normalized_input": true, |
| "siamesenorm_kernel_backend": "reference", |
| "stack_d_model": 24, |
| "stack_entropy_loss_weight": 0.001, |
| "stack_memory_cache_size": 2048, |
| "stack_slots": 16, |
| "tie_word_embeddings": false, |
| "transformers_version": "5.21.1", |
| "tweo_eps": 1e-06, |
| "tweo_loss_weight": 0.01, |
| "tweo_power": 4.0, |
| "tweo_tau": 3.0, |
| "use_affine_scaled_attention": false, |
| "use_attn_res": false, |
| "use_cache": false, |
| "use_copy_paste": true, |
| "use_dagformer": false, |
| "use_directional_routing": false, |
| "use_embedding_input_norm": true, |
| "use_embedding_multipliers": false, |
| "use_fan_residual": false, |
| "use_gpas": false, |
| "use_hadamard_o_proj": true, |
| "use_hola_memory": false, |
| "use_iha": false, |
| "use_jtok": false, |
| "use_jtokm": false, |
| "use_laurel": false, |
| "use_laurel_lr": false, |
| "use_laurel_rw": false, |
| "use_learnable_multipliers": true, |
| "use_liger_kernel": false, |
| "use_lns": true, |
| "use_lucid_attention": false, |
| "use_mea_attention": false, |
| "use_meap": true, |
| "use_mile_loss": true, |
| "use_momentum_attention": true, |
| "use_mu_loss": true, |
| "use_nextlat": false, |
| "use_nitp": true, |
| "use_nitp_temporal": false, |
| "use_perplexity_metrics": true, |
| "use_repo": true, |
| "use_repo_goat_prior": false, |
| "use_repo_grape": true, |
| "use_siamesenorm": true, |
| "use_spelling_bee_embeddings": false, |
| "use_stack_memory": false, |
| "use_token_generator": true, |
| "use_topk_metrics": true, |
| "use_tweo": true, |
| "use_vha_attention": false, |
| "use_xsa": true, |
| "vocab_size": 64402, |
| "xsa_eps": 1e-06 |
| } |
|
|
