File size: 1,567 Bytes
47f1a10 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 |
#!/bin/bash
# Start CodeLlama fine-tuning with optimized hyperparameters
cd /workspace/ftt/codellama-migration
BASE_MODEL="/workspace/ftt/codellama-migration/models/base-models/CodeLlama-7B-Instruct"
DATASET="datasets/processed/split/train.jsonl"
OUTPUT_DIR="training-outputs/codellama-fifo-v1"
echo "======================================================================"
echo "🚀 Starting CodeLlama Fine-Tuning"
echo "======================================================================"
echo "Base Model: $BASE_MODEL"
echo "Dataset: $DATASET"
echo "Output: $OUTPUT_DIR"
echo "======================================================================"
# Create output directory
mkdir -p "$OUTPUT_DIR"
# Activate virtual environment if needed
if [ -f /venv/main/bin/activate ]; then
source /venv/main/bin/activate
fi
# Start training with optimized hyperparameters
python3 scripts/training/finetune_codellama.py \
--base-model "$BASE_MODEL" \
--dataset "$DATASET" \
--output-dir "$OUTPUT_DIR" \
--resume-from-checkpoint auto \
--max-length 1536 \
--num-epochs 5 \
--batch-size 2 \
--gradient-accumulation 4 \
--learning-rate 2e-5 \
--lora-r 48 \
--lora-alpha 96 \
--lora-dropout 0.15 \
--warmup-ratio 0.1 \
--eval-steps 25 \
--save-steps 25 \
--early-stopping-patience 5 \
--logging-steps 5
echo ""
echo "======================================================================"
echo "✅ Training Complete!"
echo "======================================================================"
|