| { | |
| "model_class": "LemonEncoder", | |
| "architecture": { | |
| "vocab_size": 32000, | |
| "embed_dim": 768, | |
| "num_layers": 24, | |
| "nhead": 12, | |
| "ff_mult": 4, | |
| "dropout": 0.04, | |
| "max_tokens": 1024, | |
| "proj_dim": 384, | |
| "padding_idx": 0, | |
| "rope_scaling_type": "linear", | |
| "num_proj_layers": 1, | |
| "proj_ff_mult": 1 | |
| }, | |
| "epoch": 3, | |
| "global_step": 40371, | |
| "best_val_loss": 0.2449137058109045, | |
| "best_epoch": 2 | |
| } |