checkpoints/domofon-1b-parallel/step2900/config.json

{"model": {"d_model": 2048, "vocab_size": 49152, "n_layers": 18, "block": {"sequence_mixer": {"name": "default", "n_heads": 16, "bias": false, "rope": {"name": "default", "theta": 500000, "full_precision": true, "no_global_rope": false, "partial_rotary_factor": 1.0, "_CLASS_": "olmo_core.nn.rope.RoPEConfig"}, "qk_norm": {"name": "rms", "eps": 1e-06, "bias": false, "dtype": "bfloat16", "_CLASS_": "olmo_core.nn.layer_norm.LayerNormConfig"}, "dtype": "bfloat16", "use_head_qk_norm": false, "_CLASS_": "olmo_core.nn.attention.AttentionConfig", "type": "attention"}, "layer_norm": {"name": "rms", "eps": 1e-06, "bias": false, "dtype": "bfloat16", "_CLASS_": "olmo_core.nn.layer_norm.LayerNormConfig"}, "feed_forward": {"hidden_size": 8192, "name": "default", "bias": false, "dtype": "bfloat16", "activation": "silu", "_CLASS_": "olmo_core.nn.feed_forward.FeedForwardConfig"}, "name": "reordered_norm", "_CLASS_": "olmo_core.nn.transformer.config.TransformerBlockConfig"}, "lm_head": {"name": "default", "layer_norm": {"name": "rms", "eps": 1e-06, "bias": false, "dtype": "bfloat16", "_CLASS_": "olmo_core.nn.layer_norm.LayerNormConfig"}, "bias": false, "dtype": "bfloat16", "loss_implementation": "default", "_CLASS_": "olmo_core.nn.lm_head.LMHeadConfig"}, "name": "default", "dtype": "bfloat16", "init_method": "normal", "init_seed": 42, "init_std": 0.02, "tie_word_embeddings": false, "_CLASS_": "olmo_core.nn.transformer.config.TransformerConfig"}, "dataset": {"tokenizer": {"vocab_size": 49152, "eos_token_id": 0, "pad_token_id": 3, "bos_token_id": 0, "identifier": "HuggingFaceTB/SmolLM2-1.7B", "_CLASS_": "olmo_core.data.tokenizer.TokenizerConfig"}}}