tokenizer-experiment/gate2/sparse-smoke/gate2_sparse_smoke.json

{
  "backend": "nccl",
  "batch_size_per_rank": 128,
  "embedding_dim": 1024,
  "gradient_dense_before_adamw": true,
  "gradient_sparse_before_adamw": true,
  "local_touched_rows": 3808,
  "peak_cuda_memory_bytes": 5385528320,
  "status": "ok",
  "synchronized_touched_rows": 14959,
  "tokens_per_row": 30,
  "vocab_size": 262144,
  "world_size": 4
}