config.json

{
  "model_class": "LemonEncoder",
  "architecture": {
    "vocab_size": 32000,
    "embed_dim": 768,
    "num_layers": 24,
    "nhead": 12,
    "ff_mult": 4,
    "dropout": 0.04,
    "max_tokens": 1024,
    "proj_dim": 384,
    "padding_idx": 0,
    "rope_scaling_type": "linear",
    "num_proj_layers": 1,
    "proj_ff_mult": 1
  },
  "epoch": 3,
  "global_step": 40371,
  "best_val_loss": 0.2449137058109045,
  "best_epoch": 2
}