{
  "model_type": "halo_s",
  "architectures": [
    "HaloSForCausalLM"
  ],
  "auto_map": {
    "AutoConfig": "configuration_halos.HaloSConfig",
    "AutoModel": "modeling_halos.HaloSHFModel",
    "AutoModelForCausalLM": "modeling_halos.HaloSForCausalLM"
  },
  "vocab_size": 50257,
  "hidden_size": 512,
  "num_layers": 6,
  "num_heads": 8,
  "num_kv_heads": 2,
  "num_globals": 2,
  "local_window": 64,
  "dilated_offsets": [1, 2, 4, 8, 16, 32, 64, 128],
  "num_random": 2,
  "dropout": 0.1,
  "use_swiglu": false,
  "max_seq_len": 1024,
  "torch_dtype": "float16",
  "framework": "pyhalos",
  "architecture": "HALO-S",
  "tokenizer": "tiktoken-gpt2",
  "training": {
    "dataset": "wikitext-103-raw-v1",
    "max_tokens": 30000000,
    "epochs": 2,
    "batch_size": 4,
    "grad_accumulation": 4,
    "effective_batch": 32,
    "precision": "fp16_mixed",
    "gpus": "2x T4 (DataParallel)",
    "gradient_checkpointing": true
  }
}