| { | |
| "dataset": { | |
| "remote_path": "flpelerin/tinystories-10k", | |
| "batch_size": 4, | |
| "seq_length": 512 | |
| }, | |
| "tokenizer": { | |
| "max_length": 32000 | |
| }, | |
| "wandb": { | |
| "entity": "florianpelerin110304", | |
| "project": "mamba-trainer-test", | |
| "name": "", | |
| "api_key": "860f8753998c6e6dc356914de07e8855aa2f9642" | |
| }, | |
| "model": { | |
| "params": { | |
| "d_model": 768, | |
| "n_layer": 24, | |
| "tie_embeddings": "false" | |
| } | |
| }, | |
| "trainer": { | |
| "learning_rate": 1e-3, | |
| "num_epochs": 4, | |
| "inference": { | |
| "frequency": 0, | |
| "n_predict": 256, | |
| "seed_text": "One day, a little girl" | |
| } | |
| } | |
| } | |