config.json

{
  "model": "NeuroMamba-v5-NoFastWeight",
  "n_params": 28809768,
  "d_model": 312,
  "n_q": 6,
  "n_kv": 2,
  "d_head": 52,
  "d_ffn": 896,
  "window": 128,
  "n_local": 5,
  "n_global": 1,
  "n_cycles": 2,
  "seq_len": 512,
  "batch_size": 16,
  "grad_accum": 4,
  "lr": 0.0003,
  "warmup": 500,
  "target_tokens": 600000000,
  "tokens_seen": 599982080,
  "best_eval_loss": 4.442645943164825,
  "best_eval_perplexity": 84.99954836853307,
  "baseline_eval_loss": 4.3645,
  "delta_vs_baseline": 0.07814594316482548
}