| { | |
| "model_type": "unet_transformer", | |
| "auto_map": { | |
| "AutoConfig": "modeling_unet_transformer.PLMConfig", | |
| "AutoModelForMaskedLM": "modeling_unet_transformer.UnetTransformerModel" | |
| }, | |
| "add_att_soft_cap": false, | |
| "attention_soft_cap": null, | |
| "expansion_ratio": 4.0, | |
| "hidden_size": 640, | |
| "mlm": true, | |
| "num_att_tokens": 1026, | |
| "num_attention_heads": 20, | |
| "num_hidden_layers": 30, | |
| "p_attention": false, | |
| "sliding_window_size": 2048, | |
| "soft_logit_cap": 32.0, | |
| "tie_embeddings": false, | |
| "token_dropout": true, | |
| "transformers_version": "4.55.3", | |
| "unet": true, | |
| "vocab_size": 33 | |
| } | |