{
  "accelerator": "auto",
  "accumulate_grad_batches": 2,
  "alignment_checkpoint": "output/alignment/exp12-lr5e5-b02-b8a4-val1.3030.ckpt",
  "architectures": [
    "DotLMForCausalLM"
  ],
  "auto_map": {
    "AutoConfig": "modeling_dotlm.DotLMConfig",
    "AutoModelForCausalLM": "modeling_dotlm.DotLMForCausalLM"
  },
  "batch_size": 16,
  "bos_token_id": null,
  "context_len": 4096,
  "d_model": 768,
  "dataset_path": "dataset/reasoning.json",
  "devices": "auto",
  "dtype": "float32",
  "eos_token_id": 3,
  "gradient_clip_val": 0.5,
  "hidden_dim": 2048,
  "initializer_range": 0.02,
  "learning_rate": 5e-05,
  "max_epochs": 10,
  "max_seq_len": 768,
  "max_steps": -1,
  "model_type": "dotlm",
  "n_heads": 6,
  "n_kv_heads": 2,
  "norm_eps": 1e-06,
  "num_hidden_layers": 24,
  "num_workers": 4,
  "output_dir": "output/reasoning",
  "pad_token_id": 0,
  "precision": "bf16-mixed",
  "run_prefix": "dotlm_165m",
  "save_top_k": 5,
  "scheduler_type": "cosine_with_warmup",
  "strategy": "auto",
  "theta_base": 10000.0,
  "tie_word_embeddings": true,
  "transformers_version": "5.5.0",
  "use_cache": true,
  "use_wandb": true,
  "val_split": 0.01,
  "val_steps": 200,
  "version": "auto",
  "vocab_size": 16384,
  "wandb_project": "DotLM",
  "warmup_steps": 50,
  "weight_decay": 0.01
}