File size: 1,174 Bytes
ce88159 85b9889 ce88159 85b9889 ce88159 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 | {
"actv_fn": "gelu",
"add_token_type_emb": false,
"architectures": [
"BertBlocksForMaskedLM"
],
"attn_dropout_prob": 0.1,
"attn_out_bias": true,
"attn_proj_bias": true,
"classifier_dropout": 0.1,
"emb_dropout_prob": 0.1,
"global_attention_every_n_layers": 0,
"head_type": "mlp",
"hidden_dropout_prob": 0.1,
"hidden_size": 768,
"include_final_norm": true,
"initializer_cutoff_factor": 3.0,
"initializer_gain": 1.0,
"initializer_kind": "trunc_normal",
"initializer_range": 0.02,
"intermediate_size": 3072,
"local_attention": [
-1,
-1
],
"mask_token_id": 1,
"max_sequence_length": 512,
"mlp_in_bias": true,
"mlp_out_bias": true,
"mlp_type": "mlp",
"model_type": "bertblocks",
"norm_bias": true,
"norm_eps": 1e-12,
"norm_fn": "layer",
"norm_kind": "pre",
"norm_params": {},
"num_attention_heads": 12,
"num_blocks": 12,
"pad_token_id": 0,
"pos_emb_kind": "alibi",
"pos_emb_kwargs": {
"dim": 64,
"max_seq_len": 512
},
"problem_type": "regression",
"torch_dtype": "float32",
"transformers_version": "4.53.2",
"type_vocab_size": 1,
"unpadding": false,
"vocab_size": 28996
}
|