| { | |
| "output_dir": "./models/literary-lm", | |
| "overwrite_output_dir": true, | |
| "num_train_epochs": 10, | |
| "per_device_train_batch_size": 8, | |
| "per_device_eval_batch_size": 8, | |
| "gradient_accumulation_steps": 4, | |
| "learning_rate": 5e-5, | |
| "weight_decay": 0.01, | |
| "warmup_steps": 500, | |
| "logging_steps": 100, | |
| "save_steps": 1000, | |
| "eval_steps": 500, | |
| "save_total_limit": 3, | |
| "fp16": true, | |
| "dataloader_num_workers": 4, | |
| "load_best_model_at_end": true, | |
| "metric_for_best_model": "perplexity", | |
| "greater_is_better": false, | |
| "evaluation_strategy": "steps", | |
| "save_strategy": "steps" | |
| } | |