| { |
| "rmu_cfg": { |
| "channels": 512, |
| "ray_dim": 512, |
| "virtual_dim": 512, |
| "grid_g": 8, |
| "grid_h": 8, |
| "grid_w": 8, |
| "vocab_size": 68737, |
| "bos_token_id": 50256, |
| "eos_token_id": 50256, |
| "pad_token_id": 50257, |
| "chunk_size": 32, |
| "device": "cuda:0", |
| "dtype": "torch.float32", |
| "xmem_size": 10.0, |
| "xmem_dim": 0, |
| "xmem_axes": 6, |
| "use_dsl": true, |
| "dsl_lanes": 4, |
| "dsl_rank": 64, |
| "dsl_max_sources": 96, |
| "dsl_max_per_exec": 4, |
| "dsl_steps": 1, |
| "tau_fast": 128.0, |
| "tau_slow": 512.0 |
| }, |
| "dec_cfg": { |
| "vocab_size": 68737, |
| "hidden_dim": 512, |
| "grid_g": 8, |
| "grid_h": 8, |
| "grid_w": 8, |
| "bos_token_id": 50256, |
| "eos_token_id": 50256, |
| "pad_token_id": 50257, |
| "dtype": "torch.float32", |
| "vocab_chunk_size": 4096, |
| "use_chunked_loss": true, |
| "vector_error_lambda": 0.0, |
| "factor_dim": 0 |
| }, |
| "args": { |
| "config": "configs/deep.yaml", |
| "tokenizer_dir": "tools/gwen2_tokenizer", |
| "dataset_dir": "datasets/FSS1STR", |
| "pt": "", |
| "dataset_split": null, |
| "dataset_text_column": "auto", |
| "val_dataset": "datasets/FSS1STR", |
| "val_fraction": 0.005, |
| "val_seed": 1337, |
| "dataset_type": "string", |
| "string_tokenize_otf": true, |
| "modes": "pt,nat", |
| "ptmode": false, |
| "blindmode": false, |
| "revmode": false, |
| "fimmode": false, |
| "nat_weight": 1.75, |
| "natmode": false, |
| "ckpt_dir": "ckpt", |
| "device": "cuda:0", |
| "devices": "cudas", |
| "seq_len": 512, |
| "seq_len_mul": 8, |
| "batch_size": 5, |
| "grad_accum_steps": 5, |
| "decoder_dim": 512, |
| "factor_dim": 0, |
| "core_dim": 512, |
| "virtual_dim": 0, |
| "num_slabs": 6, |
| "tau_fast": "auto", |
| "tau_slow": "auto", |
| "core_devices": "", |
| "max_steps": 5500, |
| "lr": 0.0001, |
| "lr_min": 9e-06, |
| "dtype": "bf16", |
| "param_dtype": "fp32", |
| "dtype_acts": "fp32", |
| "grads_dtype": "bf16", |
| "optim_mem_device": "", |
| "grads_device": "", |
| "optim": "atom", |
| "optim_device": "", |
| "optim_dtype": "fp32", |
| "warmup_steps": 50, |
| "gqk_lr_scale": 0.25, |
| "gqk_warmup_steps": 15, |
| "emb_proj_lr_scale": 0.25, |
| "decoder_norm_lr_scale": 0.25, |
| "weight_decay": 0.025, |
| "learnorm_lambda": 0.05, |
| "predictor_lambda": 0.01, |
| "vector_error_lambda": 0.35, |
| "predictor_lr_scale": 1.0, |
| "grad_clip": 1.95, |
| "save_every": 500, |
| "log_every": 5, |
| "update_stats_every": 0, |
| "loss_chunk_size": 4096, |
| "checkpoint_core": 0, |
| "num_workers": 32, |
| "container_k": 128, |
| "skip_slabs": false, |
| "skip_units": 0, |
| "container_lr_scale": 0.45, |
| "cont_layers": 6, |
| "use_dsl": true, |
| "dsl_lanes": 4, |
| "dsl_rank": 64, |
| "dsl_max_sources": 96, |
| "dsl_max_sources_per_exec": 4, |
| "dsl_source_lambda": 5e-05, |
| "dsl_steps": 1, |
| "dsl_log": true, |
| "dsl_lr_scale": 0.85, |
| "dsl_exec_lambda": 0.0001, |
| "dsl_carry_lambda": 5e-05, |
| "xmem_size": 10.0, |
| "resume": "ckpt/step_0006000.pt", |
| "dtype_audit_steps": 0 |
| } |
| } |
