config.json

{
    "model_type": "mistral",
    "architectures": ["MistralForCausalLM"],
    "base_model": "mistralai/Mistral-7B-Instruct-v0.3",
    "hidden_size": 4096,
    "intermediate_size": 14336,
    "max_position_embeddings": 32768,
    "num_attention_heads": 32,
    "num_hidden_layers": 32,
    "rms_norm_eps": 1e-5,
    "rope_scaling": null,
    "rope_theta": 10000.0,
    "sliding_window": null,
    "tie_word_embeddings": false,
    "torch_dtype": "float16",
    "use_cache": true,
    "vocab_size": 32000,
    "adapter_config": {
        "r": 16,
        "lora_alpha": 32,
        "target_modules": ["q_proj", "v_proj"],
        "lora_dropout": 0.1,
        "bias": "none",
        "task_type": "CAUSAL_LM"
    },
    "training_config": {
        "checkpoint_latest": "checkpoint-36",
        "total_steps": 36,
        "optimizer": "AdamW",
        "learning_rate": 5e-5,
        "warmup_steps": 100,
        "weight_decay": 0.01
    }
}