{
    "model": {
        "max_batch_size": 16,
        "max_seq_len": 512,
        "dtype": "fp32",
        "scale_fmt": null,
        "vocab_size": 32768,
        "dim": 512,
        "inter_dim": 4096,
        "moe_inter_dim": 512,
        "n_layers": 16,
        "n_dense_layers": 3,
        "n_heads": 12,
        "n_routed_experts": 4,
        "n_shared_experts": 1,
        "n_activated_experts": 2,
        "route_scale": 1.0,
        "use_routing_bias": true,
        "q_lora_rank": 0,
        "kv_lora_rank": 256,
        "qk_nope_head_dim": 64,  
        "qk_rope_head_dim": 32,
        "v_head_dim": 64,
        "original_seq_len": 4096,
        "rope_theta": 10000.0,
        "rope_factor": 40,
        "beta_fast": 32,
        "beta_slow": 1,
        "mscale": 1.0,
        "tokenizer_name": "turkish"
    },
    "training": {
        "learning_rate": 3e-5,
        "weight_decay": 0.1,
        "beta1": 0.9,
        "beta2": 0.95,
        "grad_clip": 1.0,
        "warmup_steps": 1000,
        "total_steps": 100000,
        "use_checkpointing": false,
        "expert_rotation_steps": 5000,
        "gradient_accumulation_steps": 8,
        "eval_every": 1000,
        "save_every": 5000,
        "save_dir": "./checkpoints",
        "log_every": 100,
        "dtype": "fp32",
        "compile": false,
        "max_val_batches": 50,
        "val_batch_size_multiplier": 4,
        "train_all_experts":false
    },
    "data": {
        "train_file": "./data/train.txt",
        "val_file": "./data/val.txt",
        "stride": 512
    },
    "logging": {
        "use_wandb": true,
        "project_name": "sequential-moe",
        "run_name": "moe-12gb-gpu"
    }
}