config.json

{
  "sequence_len": 1024,
  "vocab_size": 64663,
  "n_layer": 12,
  "n_head": 6,
  "n_kv_head": 6,
  "n_embd": 768,
  "iterations": 16,
  "d_model": 384,
  "d_input": 1024,
  "heads": 8,
  "n_synch_out": 192,
  "n_synch_action": 192,
  "synapse_depth": 8,
  "memory_length": 8,
  "deep_nlms": true,
  "memory_hidden_dims": 64,
  "do_layernorm_nlm": false,
  "predict_n_tokens": 1,
  "backbone_type": "none",
  "positional_embedding_type": "none",
  "out_dims": null,
  "prediction_reshaper": [
    -1
  ],
  "dropout": 0.05,
  "dropout_nlm": null,
  "neuron_select_type": "random-pairing",
  "n_random_pairing_self": 0,
  "sub_task": "plasticDeepLatent",
  "attention": {
    "vocab_size": 64663,
    "emb_dim": 1024,
    "hidden_dim": 3072,
    "n_layers": 12,
    "n_heads": 8,
    "n_kv_groups": 4,
    "head_dim": 128,
    "context_length": 1024,
    "rope_base": 1000000.0,
    "qk_norm": true,
    "attention_dropout": 0.0,
    "attention_bias": false,
    "attn_implementation": "sdpa"
  },
  "prediction_iterations": [
    4,
    8,
    12,
    16
  ],
  "ensemble_weights": [
    0.1,
    0.2,
    0.3,
    0.4
  ],
  "iteration_residual_interval": 4,
  "iteration_residual_scale": 0.3,
  "iteration_checkpoint": true,
  "iteration_checkpoint_interval": 8,
  "ultrathink_enabled": false,
  "ultrathink_multi_head_sync": false,
  "ultrathink_sync_heads": 4,
  "ultrathink_sync_decay_rates": null,
  "ultrathink_sync_aggregation": "concat",
  "ultrathink_sync_attention": false,
  "ultrathink_sync_attention_heads": 8,
  "ultrathink_cross_sync": false,
  "ultrathink_cross_sync_heads": 4,
  "ultrathink_confidence_early_exit": false,
  "ultrathink_confidence_threshold": 0.95,
  "ultrathink_min_iterations": 16,
  "ultrathink_sync_warmup_iterations": 0,
  "ultrathink_sync_warmup_weight": 0.1,
  "exit_gate_enabled": true,
  "exit_gate_hidden_size": 0,
  "exit_gate_min_iterations": 1,
  "exit_gate_entropy_lambda": 0.01,
  "exit_gate_stage2": false,
  "exit_gate_stage2_threshold": 0.05,
  "exit_gate_stage2_steepness": 50.0,
  "no_ctm": false,
  "standard_decoder": false,
  "causal_ctm": true,
  "use_spectral_norm": true,
  "plastic_weight_clip": 1.0,
  "iteration_grad_scale": true,
  "o1_memory": true,
  "plastic_ode": true,
  "nlm_chunk_size": 1024,
  "nlm_variant": "gated",
  "chunked_ctm": true,
  "chunked_ctm_num_chunks": 4,
  "chunked_ctm_sync_interval": 8
}