jax_model/hyperparams.json

{
    "alphabet_size": 4107,
    "pad_token_id": 1,
    "mask_token_id": 2,
    "max_positions": 2048,
    "embed_scale": 1.0,
    "emb_layer_norm_before": false,
    "attention_heads": 16,
    "key_size": 64,
    "embed_dim": 1024,
    "ffn_embed_dim": 4096,
    "num_layers": 29,
    "positional_embedding": null,
    "lm_head": "roberta",
    "add_bias_kv": false,
    "add_bias_ffn": false,
    "use_rotary_embedding": true,
    "ffn_activation_name": "swish",
    "use_glu_in_ffn": true,
    "mask_before_attention": false,
    "token_dropout": false,
    "masking_ratio": 0.0,
    "masking_prob": 0.0,
    "use_gradient_checkpointing": false,
    "k_for_kmers": 6,
    "rescaling_factor": 2.44140625,
    "features": [
        "protein_coding_gene",
        "lncRNA",
        "exon",
        "intron",
        "splice_donor",
        "splice_acceptor",
        "5UTR",
        "3UTR",
        "CTCF-bound",
        "polyA_signal",
        "enhancer_Tissue_specific",
        "enhancer_Tissue_invariant",
        "promoter_Tissue_specific",
        "promoter_Tissue_invariant"
    ]
}