{
  "model": {
    "hidden_dim": 768,
    "intermediate_dim": 2048,
    "num_layers": 16,
    "num_attention_heads": 12,
    "num_key_value_heads": 4,
    "head_dim": 64,
    "rope_theta": 1000000.0,
    "max_position_embeddings": 4096,
    "num_experts": 4,
    "top_k_experts": 2,
    "expert_hidden_dim": 768,
    "moe_shared_experts": 1,
    "moe_aux_loss_coef": 0.01,
    "moe_z_loss_coef": 0.001,
    "vocab_size": 32768,
    "max_seq_len": 4096,
    "num_tool_heads": 4,
    "rope_dim": 64,
    "rms_norm_eps": 1e-05,
    "norm_eps": 1e-05,
    "padding_idx": 0,
    "eos_token_id": 2,
    "use_mtp": false,
    "num_mtp_tokens": 3,
    "use_infini_attention": false,
    "use_latent_attention": false,
    "ssm_layer_interval": 999,
    "use_xlstm": false,
    "use_differential_attention": false,
    "use_mixture_of_depth": false,
    "use_snapkv": false,
    "use_bitnet": true,
    "use_lora": false,
    "lora_r": 8,
    "lora_alpha": 16,
    "sparse_attention": false,
    "quantize_kv_cache": false,
    "ssm_state_dim": 16,
    "ssm_conv_kernel": 4,
    "xlstm_mem_dim": 64,
    "mod_top_k_frac": 0.5,
    "snapkv_cache_size": 2048,
    "snapkv_keep_first": 16,
    "snapkv_window": 256,
    "snapkv_compression": 0.25,
    "kv_compression_ratio": 8,
    "mem_momentum": 0.9,
    "mem_segment_len": 256,
    "max_segments": 4096,
    "complexity_hidden": 768,
    "complexity_layers": 2,
    "hidden_dropout": 0.0
  },
  "training": {
    "batch_size": 8,
    "learning_rate": 0.0001,
    "weight_decay": 0.0,
    "num_epochs": 4,
    "max_length": 1024,
    "gradient_accumulation_steps": 4,
    "gradient_checkpointing": true,
    "max_grad_norm": 1.0,
    "warmup_steps": 100,
    "log_every_n_steps": 10,
    "save_every_n_steps": 300,
    "output_dir": "./checkpoints",
    "dtype": "float16",
    "quantize": true,
    "quantization_method": "int8",
    "adam_beta1": 0.9,
    "adam_beta2": 0.999,
    "ewc_lambda": 0.0,
    "ewc_fisher_samples": 1024
  },
  "hackernet_version": "v1",
  "architecture": "HackerNet-Beta-v3.0",
  "framework": "PyTorch"
}