{
  "beta1": 0.9,
  "beta2": 0.95,
  "bias_correction": true,
  "ckpt_interval": 100,
  "compute_dtype": "bfloat16",
  "data_dir": "data/shards",
  "data_index": "data/shards/index.json",
  "data_integrity": {
    "receipt": "config/run1_shard_integrity_receipt.json",
    "role": "current",
    "sha256": "5831ecd4a471fbe07e19b212bc3de44bed0b0b6b456083e888f66802937bf471"
  },
  "dim": 768,
  "eps": 1e-08,
  "eval_interval": 500,
  "ffn_hidden": 2048,
  "fim_chunk": 1024,
  "fim_rate": 0.7,
  "grad_accum": 16,
  "grad_clip": 1.0,
  "log_interval": 10,
  "lr": 0.0006,
  "lr_min": 6e-05,
  "max_steps": 19073,
  "micro_batch": 8,
  "mtp_depth": 2,
  "mtp_layers": 1,
  "mtp_weight": 0.3,
  "n_heads": 12,
  "n_kv_heads": 4,
  "n_layers": 12,
  "norm_eps": 1e-05,
  "out_dir": "out/run1",
  "rope_theta": 100000.0,
  "run_name": "wisp-run1-110m-code",
  "sampler_reset_steps": [
    300
  ],
  "seed": 1337,
  "seq_len": 2048,
  "snapshot_interval": 2000,
  "sources": [
    {
      "data_dir": "python",
      "quality_gate": true,
      "repo": "bigcode/starcoderdata",
      "text_field": "content",
      "weight": 0.24
    },
    {
      "data_dir": "javascript",
      "quality_gate": true,
      "repo": "bigcode/starcoderdata",
      "text_field": "content",
      "weight": 0.13
    },
    {
      "data_dir": "typescript",
      "quality_gate": true,
      "repo": "bigcode/starcoderdata",
      "text_field": "content",
      "weight": 0.12
    },
    {
      "data_dir": "go",
      "quality_gate": true,
      "repo": "bigcode/starcoderdata",
      "text_field": "content",
      "weight": 0.09
    },
    {
      "data_dir": "rust",
      "quality_gate": true,
      "repo": "bigcode/starcoderdata",
      "text_field": "content",
      "weight": 0.09
    },
    {
      "data_dir": "java",
      "quality_gate": true,
      "repo": "bigcode/starcoderdata",
      "text_field": "content",
      "weight": 0.08
    },
    {
      "data_dir": "c",
      "quality_gate": true,
      "repo": "bigcode/starcoderdata",
      "text_field": "content",
      "weight": 0.05
    },
    {
      "data_dir": "shell",
      "quality_gate": true,
      "repo": "bigcode/starcoderdata",
      "text_field": "content",
      "weight": 0.03
    },
    {
      "data_dir": "sql",
      "quality_gate": true,
      "repo": "bigcode/starcoderdata",
      "text_field": "content",
      "weight": 0.02
    },
    {
      "data_dir": "markdown",
      "quality_gate": true,
      "repo": "bigcode/starcoderdata",
      "text_field": "content",
      "weight": 0.07
    },
    {
      "name": "sample-10BT",
      "quality_gate": true,
      "repo": "HuggingFaceFW/fineweb-edu",
      "text_field": "text",
      "weight": 0.08
    }
  ],
  "tie_embeddings": true,
  "tokenizer_path": "tokenizer/code32k.json",
  "vocab_size": 32768,
  "warmup_steps": 1000,
  "weight_decay": 0.1
}