config.json

{
  "model": "sphere-ff",
  "dim": 512,
  "depth": 8,
  "heads": 8,
  "seq_len": 512,
  "batch_size": 32,
  "steps": 60000,
  "warmup": 1000,
  "lr": 0.0006,
  "solver_iters": 4,
  "solver_tol": 0.001,
  "eval_every": 1000,
  "seed": 0,
  "grad_accum": 2,
  "no_bf16": false,
  "r_init": 0.7,
  "logit_scale": 10.0,
  "tag": "clufull1",
  "save_checkpoint": true,
  "checkpoint_every": 0,
  "s3_prefix": "",
  "data_dir": "data1b",
  "wandb": false,
  "wandb_project": "spherical-attention",
  "wandb_entity": null,
  "cloudwatch": false,
  "cloudwatch_region": "us-east-2",
  "resume": "",
  "ngpt": true,
  "selective_wd": true,
  "grad_steps": 1,
  "ema": 0.999,
  "cluster_reg": 0.1,
  "abort_divergence": 1.5
}