| { |
| "model": { |
| "hidden_dim": 768, |
| "intermediate_dim": 2048, |
| "num_layers": 16, |
| "num_attention_heads": 12, |
| "num_key_value_heads": 4, |
| "head_dim": 64, |
| "rope_theta": 1000000.0, |
| "max_position_embeddings": 4096, |
| "num_experts": 4, |
| "top_k_experts": 2, |
| "expert_hidden_dim": 768, |
| "moe_shared_experts": 1, |
| "moe_aux_loss_coef": 0.01, |
| "moe_z_loss_coef": 0.001, |
| "vocab_size": 32768, |
| "max_seq_len": 4096, |
| "num_tool_heads": 4, |
| "rope_dim": 64, |
| "rms_norm_eps": 1e-05, |
| "norm_eps": 1e-05, |
| "padding_idx": 0, |
| "eos_token_id": 2, |
| "use_mtp": false, |
| "num_mtp_tokens": 3, |
| "use_infini_attention": false, |
| "use_latent_attention": false, |
| "ssm_layer_interval": 999, |
| "use_xlstm": false, |
| "use_differential_attention": false, |
| "use_mixture_of_depth": false, |
| "use_snapkv": false, |
| "use_bitnet": true, |
| "use_lora": false, |
| "lora_r": 8, |
| "lora_alpha": 16, |
| "sparse_attention": false, |
| "quantize_kv_cache": false, |
| "ssm_state_dim": 16, |
| "ssm_conv_kernel": 4, |
| "xlstm_mem_dim": 64, |
| "mod_top_k_frac": 0.5, |
| "snapkv_cache_size": 2048, |
| "snapkv_keep_first": 16, |
| "snapkv_window": 256, |
| "snapkv_compression": 0.25, |
| "kv_compression_ratio": 8, |
| "mem_momentum": 0.9, |
| "mem_segment_len": 256, |
| "max_segments": 4096, |
| "complexity_hidden": 768, |
| "complexity_layers": 2, |
| "hidden_dropout": 0.0 |
| }, |
| "training": { |
| "batch_size": 8, |
| "learning_rate": 0.0001, |
| "weight_decay": 0.0, |
| "num_epochs": 4, |
| "max_length": 1024, |
| "gradient_accumulation_steps": 4, |
| "gradient_checkpointing": true, |
| "max_grad_norm": 1.0, |
| "warmup_steps": 100, |
| "log_every_n_steps": 10, |
| "save_every_n_steps": 300, |
| "output_dir": "./checkpoints", |
| "dtype": "float16", |
| "quantize": true, |
| "quantization_method": "int8", |
| "adam_beta1": 0.9, |
| "adam_beta2": 0.999, |
| "ewc_lambda": 0.0, |
| "ewc_fisher_samples": 1024 |
| }, |
| "hackernet_version": "v1", |
| "architecture": "HackerNet-Beta-v3.0", |
| "framework": "PyTorch" |
| } |
