{
  "rmu_cfg": {
    "channels": 512,
    "ray_dim": 512,
    "virtual_dim": 512,
    "grid_g": 8,
    "grid_h": 8,
    "grid_w": 8,
    "vocab_size": 68737,
    "bos_token_id": 50256,
    "eos_token_id": 50256,
    "pad_token_id": 50257,
    "chunk_size": 32,
    "device": "cuda:0",
    "dtype": "torch.float32",
    "xmem_size": 10.0,
    "xmem_dim": 0,
    "xmem_axes": 6,
    "use_dsl": true,
    "dsl_lanes": 4,
    "dsl_rank": 64,
    "dsl_max_sources": 96,
    "dsl_max_per_exec": 4,
    "dsl_steps": 1,
    "tau_fast": 128.0,
    "tau_slow": 512.0
  },
  "dec_cfg": {
    "vocab_size": 68737,
    "hidden_dim": 512,
    "grid_g": 8,
    "grid_h": 8,
    "grid_w": 8,
    "bos_token_id": 50256,
    "eos_token_id": 50256,
    "pad_token_id": 50257,
    "dtype": "torch.float32",
    "vocab_chunk_size": 4096,
    "use_chunked_loss": true,
    "vector_error_lambda": 0.0,
    "factor_dim": 0
  },
  "args": {
    "config": "configs/deep.yaml",
    "tokenizer_dir": "tools/gwen2_tokenizer",
    "dataset_dir": "datasets/FSS1STR",
    "pt": "",
    "dataset_split": null,
    "dataset_text_column": "auto",
    "val_dataset": "datasets/FSS1STR",
    "val_fraction": 0.005,
    "val_seed": 1337,
    "dataset_type": "string",
    "string_tokenize_otf": true,
    "modes": "pt,nat",
    "ptmode": false,
    "blindmode": false,
    "revmode": false,
    "fimmode": false,
    "nat_weight": 1.75,
    "natmode": false,
    "ckpt_dir": "ckpt",
    "device": "cuda:0",
    "devices": "cudas",
    "seq_len": 512,
    "seq_len_mul": 8,
    "batch_size": 5,
    "grad_accum_steps": 5,
    "decoder_dim": 512,
    "factor_dim": 0,
    "core_dim": 512,
    "virtual_dim": 0,
    "num_slabs": 6,
    "tau_fast": "auto",
    "tau_slow": "auto",
    "core_devices": "",
    "max_steps": 5500,
    "lr": 0.0001,
    "lr_min": 9e-06,
    "dtype": "bf16",
    "param_dtype": "fp32",
    "dtype_acts": "fp32",
    "grads_dtype": "bf16",
    "optim_mem_device": "",
    "grads_device": "",
    "optim": "atom",
    "optim_device": "",
    "optim_dtype": "fp32",
    "warmup_steps": 50,
    "gqk_lr_scale": 0.25,
    "gqk_warmup_steps": 15,
    "emb_proj_lr_scale": 0.25,
    "decoder_norm_lr_scale": 0.25,
    "weight_decay": 0.025,
    "learnorm_lambda": 0.05,
    "predictor_lambda": 0.01,
    "vector_error_lambda": 0.35,
    "predictor_lr_scale": 1.0,
    "grad_clip": 1.95,
    "save_every": 500,
    "log_every": 5,
    "update_stats_every": 0,
    "loss_chunk_size": 4096,
    "checkpoint_core": 0,
    "num_workers": 32,
    "container_k": 128,
    "skip_slabs": false,
    "skip_units": 0,
    "container_lr_scale": 0.45,
    "cont_layers": 6,
    "use_dsl": true,
    "dsl_lanes": 4,
    "dsl_rank": 64,
    "dsl_max_sources": 96,
    "dsl_max_sources_per_exec": 4,
    "dsl_source_lambda": 5e-05,
    "dsl_steps": 1,
    "dsl_log": true,
    "dsl_lr_scale": 0.85,
    "dsl_exec_lambda": 0.0001,
    "dsl_carry_lambda": 5e-05,
    "xmem_size": 10.0,
    "resume": "ckpt/step_0006000.pt",
    "dtype_audit_steps": 0
  }
}