| { | |
| "model": { | |
| "embedding_config": { | |
| "context_size": 1024, | |
| "vocab_size": 32000, | |
| "d_model": 768, | |
| "dropout_prob": 0.1 | |
| }, | |
| "transformer_config": { | |
| "mha_config": { | |
| "d_model": 768, | |
| "num_heads": 12, | |
| "dropout_prob": 0.1 | |
| }, | |
| "ff_expansion_factor": 4, | |
| "activation": "Gelu", | |
| "dropout_prob": 0.1 | |
| }, | |
| "num_transformer_layers": 12 | |
| }, | |
| "optimizer": { | |
| "grad_clipping": null, | |
| "beta_1": 0.9, | |
| "beta_2": 0.999, | |
| "epsilon": 0.00001, | |
| "weight_decay": 0.0001, | |
| "cautious_weight_decay": false, | |
| "amsgrad": false | |
| }, | |
| "tokenizer_kind": "bpe", | |
| "tokenizer_path": "tokenizer.json", | |
| "num_epochs": 200, | |
| "batch_size": 6, | |
| "num_workers": 6, | |
| "seed": 123456, | |
| "learning_rate": 0.0001 | |
| } |