| { | |
| "model_name": "seqlens-v2-micro-16k", | |
| "architecture": "BiMamba2 + sliding-window attention", | |
| "parameters": 10347977, | |
| "d_model": 256, | |
| "n_layers": 8, | |
| "max_seq_len": 16384, | |
| "vocab_size": 9, | |
| "ssm_d_state": 64, | |
| "ssm_expand": 2, | |
| "attn_heads": 4, | |
| "attn_window": 512, | |
| "attn_layer_interval": 4, | |
| "ffn_expand": 4, | |
| "training_steps": 10000, | |
| "tokens_seen": 182766749, | |
| "training_data": "OpenGenome2 GTDB v220 (113,379 prokaryotic genomes)", | |
| "pre_training_objective": "MLM (15% mask rate)", | |
| "optimizer": "AdamW (lr=1e-3, \u03b21=0.9, \u03b22=0.98, wd=0.1)", | |
| "precision": "BF16", | |
| "hardware": "8\u00d7 NVIDIA A100-SXM4-80GB", | |
| "training_time_minutes": 57.2, | |
| "framework": "PyTorch 2.6.0 + mamba-ssm 2.2.4" | |
| } |