dtype: torch.bfloat16
implementation: transformer_lens
model_name: default
n_layers: '2'
model_seed: '1'
d_model: '4'
n_ctx: '1024'
d_head: '2'
n_heads: '8'
act_fn: gelu
d_vocab: '5000'
use_local_attn: 'False'
tokenizer_name: timaeus/TinyStories-tokenizer-5k
window_size: None
attn_types: None
attn_only: 'True'
positional_embedding_type: shortformer