config.json

{
  "model_config": {
    "model_name": "OpenLLM-Small-9k",
    "model_size": "small",
    "vocab_size": 32000,
    "n_layer": 6,
    "n_head": 8,
    "n_embd": 512,
    "block_size": 1024,
    "dropout": 0.1,
    "bias": true,
    "training_info": {
      "step": 9000,
      "best_loss": 5.177438259124756,
      "model_type": "gpt-small"
    }
  },
  "tokenizer_config": {
    "type": "sentencepiece",
    "vocab_size": 32000,
    "model_file": "data/tokenizer/tokenizer.model"
  },
  "training_config": {
    "learning_rate": 0.0001,
    "batch_size": 4,
    "gradient_accumulation_steps": 4,
    "max_steps": 9000,
    "warmup_steps": 100,
    "weight_decay": 0.01
  }
}