config.json

{
    "architectures": [
        "SykoLLM_CMN"
    ],
    "model_type": "sykollm",
    "vocab_size": 32000,
    "d_model": 768,
    "n_heads": 6,
    "n_layers": 24,
    "chunk_size": 128,
    "intermediate_size": 3072,
    "num_memory_tokens": 16,
    "context_size": 1024,
    "overlap_size": 16,
    "code_overlap_size": 64,
    "abstract_head_hidden": 256,
    "abstract_head_layers": 2,
    "token_loss_weight": 0.7,
    "abstract_loss_weight": 0.3,
    "bos_token_id": 2,
    "eos_token_id": 3,
    "pad_token_id": 0
}