configs.json

{
  "ruq-0.7m": {
    "vocab_size": 8192,
    "d_model": 64,
    "n_layers": 4,
    "n_heads": 4,
    "n_kv_heads": 4,
    "ffn_hidden": 192,
    "max_seq_len": 512,
    "rope_theta": 10000.0,
    "norm_eps": 1e-05,
    "dropout": 0.0,
    "tie_embeddings": true
  },
  "ruq-2m": {
    "vocab_size": 8192,
    "d_model": 128,
    "n_layers": 4,
    "n_heads": 4,
    "n_kv_heads": 4,
    "ffn_hidden": 384,
    "max_seq_len": 512,
    "rope_theta": 10000.0,
    "norm_eps": 1e-05,
    "dropout": 0.0,
    "tie_embeddings": true
  },
  "ruq-5m": {
    "vocab_size": 8192,
    "d_model": 256,
    "n_layers": 4,
    "n_heads": 4,
    "n_kv_heads": 4,
    "ffn_hidden": 704,
    "max_seq_len": 512,
    "rope_theta": 10000.0,
    "norm_eps": 1e-05,
    "dropout": 0.0,
    "tie_embeddings": true
  },
  "ruq-15m": {
    "vocab_size": 8192,
    "d_model": 384,
    "n_layers": 6,
    "n_heads": 6,
    "n_kv_heads": 6,
    "ffn_hidden": 1024,
    "max_seq_len": 512,
    "rope_theta": 10000.0,
    "norm_eps": 1e-05,
    "dropout": 0.0,
    "tie_embeddings": true
  },
  "ruq-30m": {
    "vocab_size": 8192,
    "d_model": 512,
    "n_layers": 8,
    "n_heads": 8,
    "n_kv_heads": 8,
    "ffn_hidden": 1408,
    "max_seq_len": 512,
    "rope_theta": 10000.0,
    "norm_eps": 1e-05,
    "dropout": 0.0,
    "tie_embeddings": true
  }
}