Update config/training/model_training.ini

config/training/model_training.ini

@@ -1,21 +1,21 @@
 [ModelTrainingConfig]
-block_size = 128
+block_size = 256
 d_model = 256
 n_layer = 4
 n_head = 4
 d_ff = 1024
 dropout = 0.1
-batch_size = 8
+batch_size = 16
 grad_accum_steps = 2
-epochs = 20
-max_lr = 3e-4
-min_lr = 3e-5
+epochs = 40
+max_lr = 2e-4
+min_lr = 2e-5
 warmup_ratio = 0.05
-weight_decay = 0.1
+weight_decay = 0.05
 grad_clip = 1.0
-label_smoothing = 0.05
+label_smoothing = 0.0
 
 [AfterTrainingConfig]
 max_new_tokens = 60
-temperature = 0.85
-top_k = 40
+temperature = 0.40
+top_k = 20