config.json

{
  "architectures": [
    "DatForCausalLM"
  ],
  "auto_map": {
    "AutoConfig": "configuration_dat.DatConfig",
    "AutoModel": "modeling_dat.DatModel",
    "AutoModelForCausalLM": "modeling_dat.DatForCausalLM",
    "AutoModelForMaskedLM": "modeling_dat.DatForMaskedLM",
    "AutoModelForSequenceClassification": "modeling_dat.DatForSequenceClassification"
  },
  "bos_token_id": 1,
  "dff_factor": 4,
  "dropout": 0.1,
  "dtype": "float32",
  "eos_token_id": 2,
  "ffn_activation": "swiglu",
  "ffn_hidden_dim_mode": "dff_factor",
  "hidden_dim": 768,
  "init_range": 0.15,
  "init_scheme": "normal_0_02_scaled_projection",
  "max_position_embeddings": 514,
  "max_rel_pos": 512,
  "max_seq_len": 514,
  "mlm_head_enabled": false,
  "model_type": "dat",
  "n_heads_ra": 3,
  "n_heads_sa": 9,
  "n_layers": 18,
  "n_symbols": null,
  "norm_first": true,
  "norm_type": "layernorm",
  "pad_token_id": 3,
  "pe_type": "rope",
  "positional_symbols_sinusoidal": false,
  "ra_n_relations": null,
  "ra_rel_activation": "identity",
  "ra_symmetric_rels": false,
  "ra_type": "rca",
  "relative_symbols_rope": true,
  "relsymbolic_dropout": 0.0,
  "relsymbolic_include_self": false,
  "relsymbolic_neighborhood_size": 2,
  "relsymbolic_normalize_rels": true,
  "relsymbolic_rel_n_heads": 4,
  "relsymbolic_rel_scale": null,
  "relsymbolic_symbolic_attn_n_heads": 4,
  "relsymbolic_symbolic_attn_scale": null,
  "relsymbolic_trainable_symbols": true,
  "relsymbolic_use_bias": false,
  "rope_theta": 10000.0,
  "segment_boundary_token_id": null,
  "sequence_boundary_policy": "document_boundary",
  "share_attn_params": false,
  "shared_symbol_retriever": true,
  "symbol_dim": null,
  "symbol_retrieval": "relative",
  "symbolic_attn_n_heads": null,
  "symbolic_use_bias": false,
  "tie_lm_head": false,
  "tie_word_embeddings": false,
  "transformers_version": "5.12.0",
  "use_bias_ffn": true,
  "use_bias_out": false,
  "use_bias_qkv": false,
  "vocab_size": 16384
}