| { | |
| "d_model": 32, | |
| "n_layer": 2, | |
| "d_inner": 128, | |
| "vocab_size": 12, | |
| "resid_dropout": 0.0, | |
| "embed_dropout": 0.1, | |
| "layer_norm_epsilon": 1e-05, | |
| "pad_vocab_size_multiple": 8, | |
| "residual_in_fp32": true, | |
| "checkpoint_mixer": false, | |
| "checkpoint_mlp": false, | |
| "max_position_embeddings": 1024, | |
| "layer": { | |
| "emb_dim": 5, | |
| "filter_order": 64, | |
| "short_filter_order": 3, | |
| "l_max": 1026, | |
| "modulate": true, | |
| "w": 10 | |
| }, | |
| "model_type": "hyenadna" | |
| } |