File size: 753 Bytes
0f1b4e6 9f9ca20 0f1b4e6 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 | {
"n_layers": 2,
"d_model": 736,
"n_ctx": 1024,
"d_head": 64,
"n_heads": 11,
"d_mlp": 2944,
"act_fn": "solu_ln",
"d_vocab": 50278,
"eps": 1e-05,
"use_attn_result": false,
"use_attn_scale": true,
"use_local_attn": false,
"model_family": null,
"checkpoint": null,
"tokenizer_name": "EleutherAI/gpt-neox-20b",
"window_size": null,
"attn_types": null,
"init_mode": "gpt2",
"normalization_type": "LNPre",
"device": "cuda",
"attention_dir": "causal",
"attn_only": false,
"seed": 42,
"initializer_range": 0.02948839123097943,
"init_weights": true,
"scale_attn_by_inverse_layer_idx": false,
"positional_embedding_type": "standard",
"final_rms": true,
"d_vocab_out": 50278,
"parallel_attn_mlp": false
} |