{
  "model_name": "WebCoder-100M",
  "vocab_size": 28672,
  "hidden_size": 768,
  "intermediate_size": 2048,
  "num_hidden_layers": 11,
  "num_attention_heads": 12,
  "num_key_value_heads": 12,
  "max_position_embeddings": 2048,
  "seq_len": 1024,
  "docs_per_language": 10000,
  "tokenizer_docs_per_language": 4000,
  "max_chars_per_doc": 200000,
  "micro_batch_size": 8,
  "grad_accum_steps": 4,
  "pretrain_lr": 0.0003,
  "sft_lr": 8e-05,
  "weight_decay": 0.1,
  "grad_clip": 1.0,
  "max_total_tokens": 2000000000,
  "pretrain_token_cap": 1950000000,
  "sft_token_cap": 50000000,
  "pretrain_warmup_tokens": 2000000,
  "pretrain_decay_tokens": 250000000,
  "sft_warmup_tokens": 250000,
  "sft_decay_tokens": 25000000,
  "train_minutes": 55.0,
  "reserve_sft_minutes": 10.0,
  "checkpoint_every_updates": 500,
  "log_every_updates": 20,
  "sft_dataset": "iamtarun/code_instructions_120k_alpaca",
  "sft_max_examples": 40000,
  "sft_epochs": 2,
  "work_dir": "/content/webcoder100m",
  "use_google_drive": false,
  "drive_dir": "/content/drive/MyDrive/WebCoder100M",
  "hf_repo_name": "webcoder-100m-html-css-js",
  "hf_private": false
}