| { | |
| "architecture": "BCLM1Model", | |
| "model": { | |
| "vocab_size": 32768, | |
| "tokenizer": "tokenmonster:english-32000-consistent-v1", | |
| "embed_dim": 384, | |
| "n_layers": 12, | |
| "max_seq_len": 16384, | |
| "dropout": 0.0, | |
| "attn_heads": 6, | |
| "attn_kv_heads": 2, | |
| "local_attn_layers": [ | |
| 1, | |
| 5, | |
| 7, | |
| 11 | |
| ], | |
| "global_attn_layers": [ | |
| 3, | |
| 9 | |
| ], | |
| "attn_window_size": 1024, | |
| "conv_kernel_size": 4, | |
| "osc_n_pairs": 1, | |
| "osc_n_real": 16, | |
| "osc_clamp_min_decay": 1e-05, | |
| "bigram_table_factor": 5 | |
| } | |
| } |