File size: 329 Bytes
53f3b70 7261976 53f3b70 7261976 53f3b70 7261976 53f3b70 7261976 53f3b70 7261976 53f3b70 7261976 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 | [ModelTrainingConfig]
block_size = 256
d_model = 256
n_layer = 4
n_head = 4
d_ff = 1024
dropout = 0.1
batch_size = 16
grad_accum_steps = 2
epochs = 40
max_lr = 2e-4
min_lr = 2e-5
warmup_ratio = 0.05
weight_decay = 0.05
grad_clip = 1.0
label_smoothing = 0.0
[AfterTrainingConfig]
max_new_tokens = 60
temperature = 0.40
top_k = 20 |