File size: 4,658 Bytes
e6fad38 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 | #!/bin/bash
scripts=(
"evaluate_medqa_raw_vs_finetuned.py"
"evaluate_gsm8k_raw_vs_finetuned.py"
"evaluate_aime_raw_vs_finetuned.py"
"evaluate_aimo_raw_vs_finetuned.py"
"evaluate_art_raw_vs_finetuned.py"
"evaluate_copa_raw_vs_finetuned_guess_effect.py"
"evaluate_goEmotion_raw_vs_finetuned.py"
)
BASE_RESULTS_DIR="/home/msalimi/users/Nima/AbductiveReasoning/GRPO/results"
RAW_MODEL_PATH="/home/msalimi/PLLMS/unsloth-Qwen2.5-14B-Instruct-unsloth-bnb-4bit"
# RAW_MODEL_PATH="/home/moein_salimi/PLLMS/unsloth-Qwen2.5-14B-Instruct-bnb-4bit"
RUN_NAME="dt11.26.15:08_e20_unsloth_Qwen2.5_14B_Instruct_bnb_4bit_bnb_4bit_lr1e-05_t0.7_ε0.2_r64_b4"
# RUN_NAME="dt11.23.10:54_e20_unsloth_Qwen2.5_14B_Instruct_bnb_4bit_bnb_4bit_lr1e-05_t0.7_ε0.2_r64_b8"
TRAINING_DIR="$BASE_RESULTS_DIR/Training_${RUN_NAME}"
FINAL_DIR="$BASE_RESULTS_DIR/${RUN_NAME}"
if [ -d "$TRAINING_DIR/checkpoint" ]; then
CHECKPOINT_DIR="$TRAINING_DIR/checkpoint"
TRAINING_BASE="$TRAINING_DIR"
elif [ -d "$FINAL_DIR/checkpoint" ]; then
CHECKPOINT_DIR="$FINAL_DIR/checkpoint"
TRAINING_BASE="$FINAL_DIR"
else
echo "ERROR: Could not find checkpoint directory."
echo "Tried:"
echo " $TRAINING_DIR/checkpoint"
echo " $FINAL_DIR/checkpoint"
exit 1
fi
echo "Using checkpoint directory: $CHECKPOINT_DIR"
echo
COMMON_ARGS="--cuda_device 0 --evaluate_checkpoints 1"
declare -A BATCH_SIZES=(
["evaluate_medqa_raw_vs_finetuned.py"]=64
["evaluate_gsm8k_raw_vs_finetuned.py"]=128
["evaluate_aime_raw_vs_finetuned.py"]=256
["evaluate_aimo_raw_vs_finetuned.py"]=64
["evaluate_art_raw_vs_finetuned.py"]=256
["evaluate_copa_raw_vs_finetuned_guess_effect.py"]=256
["evaluate_goEmotion_raw_vs_finetuned.py"]=128
)
export TRAINING_BASE
# --- SYNCHRONIZATION CONFIGURATION ---
EXCEL_PATH="./GRPO/Evaluation/metrics_summary.xlsx"
CREDENTIALS_FILE="gen-lang-client-0687240279-843f4d194021.json"
BASE_MODEL_NAME="qwen2.5-14B"
# Set the FIXED name of your Google Sheet document here
GOOGLE_SHEET_TITLE="GRPO Final Metrics Report 14B (Master Document)"
# FOLDER_ID="YOUR_GOOGLE_DRIVE_FOLDER_ID" # Uncomment and set if needed
# --- 1. INITIAL DOWNLOAD STEP: Fetch the existing sheet content ---
echo "--- 1. Initializing Local Excel from Google Sheets ---"
python3 download_sheet.py \
--excel_path "$EXCEL_PATH" \
--run_name "$RUN_NAME" \
--credentials "$CREDENTIALS_FILE" \
--google_sheet_name "$GOOGLE_SHEET_TITLE"
# --folder_id "$FOLDER_ID" # Uncomment if used
echo "-------------------------------------"
exit
# --- EVALUATION LOOP ---
for ckpt_name in $(ls -1 "$CHECKPOINT_DIR" | grep '^checkpoint-' | sort -t- -k2,2n); do
ckpt="$CHECKPOINT_DIR/$ckpt_name"
[ -d "$ckpt" ] || continue
echo "====================================="
echo "Using checkpoint: $ckpt"
echo "====================================="
for script in "${scripts[@]}"; do
batch_size="${BATCH_SIZES[$script]:-256}"
echo "Running $script with checkpoint $ckpt (batch_size=$batch_size) ..."
# FIX: Added missing backslash '\' for line continuation
python3 GRPO/Evaluation/"$script" \
$COMMON_ARGS \
--batch_size "$batch_size" \
--checkpoint_path "$ckpt" \
--run "$RUN_NAME" \
--raw_path "$RAW_MODEL_PATH"
echo "Finished $script"
echo "-------------------------------------"
echo "--- Fetching Excel from Google Sheets ---"
python3 download_sheet.py \
--excel_path "$EXCEL_PATH" \
--run_name "$RUN_NAME" \
--credentials "$CREDENTIALS_FILE" \
--google_sheet_name "$GOOGLE_SHEET_TITLE"
# --folder_id "$FOLDER_ID" # Uncomment if used
echo "-------------------------------------"
# 2. UPDATE LOCAL EXCEL: Reads evaluation results and merges/updates the local Excel file
python3 GRPO/Evaluation/create_table.py \
--root "./GRPO/Evaluation/" \
--out_csv "$EXCEL_PATH" \
--run "$RUN_NAME" \
--base_model_name "$BASE_MODEL_NAME"
# 3. UPLOAD/SYNC: Uploads the updated local Excel file back to the specified Google Sheet
echo "--- Syncing updated Excel to Google Sheets ---"
python3 upload_sheet.py \
--excel_path "$EXCEL_PATH" \
--run_name "$RUN_NAME" \
--credentials "$CREDENTIALS_FILE" \
--google_sheet_name "$GOOGLE_SHEET_TITLE"
# --folder_id "$FOLDER_ID" # Uncomment if used
echo "-------------------------------------"
done
done |