000000000500/config.json

{
    "type": "slither.SlitherModel",
    "pretrained_url": null,
    "pretrained_step": null,
    "pretrained_strict": null,
    "torch_dtype": "float32",
    "vocab_size": 49153,
    "bos_token_id": 0,
    "eos_token_id": 0,
    "pad_token_id": 49152,
    "hidden_size": 1024,
    "num_hidden_layers": 16,
    "num_attention_heads": 8,
    "num_key_value_heads": 4,
    "intermediate_size": 1664,
    "hidden_act": "silu",
    "max_position_embeddings": 4096,
    "rope_theta": 10000,
    "rms_norm_eps": 1e-05,
    "attention_kernel": "flash_attention",
    "pure_modules": [
        "SlitherStateWriter"
    ],
    "position_embedding_dim": 256,
    "num_output_layers": 4,
    "num_memory_layers": 4,
    "state_size": 512,
    "num_state_heads": 16,
    "init_state_out_scale": 0.1,
    "write_mixer_kernel_size": 5,
    "write_mixer_init_scale": 0.1,
    "chunk_length": 1024,
    "sharding": {
        "replicate_default": true,
        "backbone_layers.layers.*": [
            [
                "data",
                "fsdp"
            ],
            null,
            null
        ],
        "output_layers.layers.*": [
            [
                "data",
                "fsdp"
            ],
            null,
            null
        ],
        "memory_layers.layers.*": [
            [
                "data",
                "fsdp"
            ],
            null,
            null
        ],
        "lm_head": [
            [
                "data",
                "fsdp"
            ],
            null,
            null
        ]
    },
    "remat": {
        "advanced": [
            {
                "name": "backbone_layers",
                "settings": {
                    "activation_checkpoint_layers": [
                        "SlitherBackboneLayer"
                    ],
                    "optimization_barrier_layers": [
                        "SlitherBackboneLayer"
                    ],
                    "scan_layers": "layers",
                    "is_layer_pure": true
                }
            },
            {
                "name": "output_layers",
                "settings": {
                    "activation_checkpoint_layers": [
                        "SlitherOutputLayer"
                    ],
                    "optimization_barrier_layers": [
                        "SlitherOutputLayer"
                    ],
                    "scan_layers": "layers",
                    "is_layer_pure": true
                }
            },
            {
                "name": "memory_layers",
                "settings": {
                    "activation_checkpoint_layers": [
                        "SlitherMemoryLayer"
                    ],
                    "optimization_barrier_layers": [
                        "SlitherMemoryLayer"
                    ],
                    "scan_layers": "layers",
                    "is_layer_pure": true
                }
            }
        ]
    }
}