| [ModelTrainingConfig] | |
| block_size = 256 | |
| d_model = 256 | |
| n_layer = 4 | |
| n_head = 4 | |
| d_ff = 1024 | |
| dropout = 0.1 | |
| batch_size = 16 | |
| grad_accum_steps = 2 | |
| epochs = 40 | |
| max_lr = 2e-4 | |
| min_lr = 2e-5 | |
| warmup_ratio = 0.05 | |
| weight_decay = 0.05 | |
| grad_clip = 1.0 | |
| label_smoothing = 0.0 | |
| [AfterTrainingConfig] | |
| max_new_tokens = 60 | |
| temperature = 0.40 | |
| top_k = 20 |