File size: 263 Bytes
a0ca968 | 1 2 3 4 5 6 7 8 9 10 11 12 | {
"attention_backend": "sdpa",
"compute_dtype": "torch.bfloat16",
"n_embd": 768,
"n_head": 6,
"n_kv_head": 6,
"n_layer": 12,
"sequence_len": 2048,
"use_activation_checkpointing": true,
"vocab_size": 16384,
"window_pattern": "SSSL"
} |