{
  "model": "Qwen/Qwen2.5-1.5B-Instruct",
  "dataset": "beAnalytic/eda-training-dataset",
  "output_dir": "beAnalytic/eda-llm-qwen2.5-lora",
  "tensorboard": {
    "report_to": ["tensorboard"],
    "logging_dir_env": "TENSORBOARD_LOGDIR",
    "run_name_env": "TRAINING_RUN_NAME",
    "launch_local": "python scripts/launch_tensorboard.py --logdir ./results"
  },
  "memory_defaults": {
    "MAX_SEQ_LENGTH": 256,
    "PER_DEVICE_TRAIN_BATCH_SIZE": 1,
    "PER_DEVICE_EVAL_BATCH_SIZE": 1,
    "GRADIENT_ACCUMULATION_STEPS": 8,
    "gradient_checkpointing": true
  },
  "sanity_gate": {
    "MIN_TRAIN_SAMPLES_env": "MIN_TRAIN_SAMPLES",
    "ALLOW_TINY_DATASET_env": "ALLOW_TINY_DATASET",
    "recommended_min_train_samples": 300
  },
  "training_config": {
    "num_train_epochs": 3,
    "per_device_train_batch_size": 4,
    "per_device_eval_batch_size": 4,
    "learning_rate": 3e-05,
    "warmup_steps": 100,
    "logging_steps": 10,
    "save_steps": 500,
    "evaluation_strategy": "steps",
    "eval_steps": 500,
    "save_total_limit": 3,
    "load_best_model_at_end": true,
    "metric_for_best_model": "loss",
    "greater_is_better": false,
    "fp16": true,
    "gradient_accumulation_steps": 2,
    "max_steps": -1
  },
  "peft_config": {
    "use_peft": true,
    "peft_method": "lora",
    "lora_r": 16,
    "lora_alpha": 32,
    "lora_dropout": 0.1,
    "target_modules": [
      "q_proj",
      "v_proj",
      "k_proj",
      "o_proj"
    ]
  },
  "quantization": {
    "use_4bit": true,
    "bnb_4bit_compute_dtype": "float16",
    "bnb_4bit_quant_type": "nf4",
    "bnb_4bit_use_double_quant": true
  }
}