eval/run_eval.sh

#!/bin/bash
# =============================================================
# VoRA Evaluation Runner (打包版)
# =============================================================
#
# ========== 使用教程 ==========
#
# 1. 把 eval_pack/ 整个文件夹传到目标机器
#
# 2. 创建 conda 环境 (一次性):
#      conda create -n eval python=3.10 -y
#      conda activate eval
#      cd eval_pack
#      pip install -r requirements_eval.txt
#
# 3. 修改下方「路径配置」部分:
#      - QWEN_MODEL:      Qwen2.5-7B-Instruct 路径
#      - QWEN_VL_MODEL:   Qwen2.5-VL-3B-Instruct 路径
#      - AIMV2_PATH:      aimv2-huge-patch14-448 路径
#      - VORA_CIRC_MERGED: VoRA-Circulant merged 模型路径
#
# 4. 修改 data/eval_qwenvl.jsonl 中的图片绝对路径:
#      sed -i 's|/share/home/jcdl1lsy2clx/VoRA/data_dir/VoRA-Recap-29M/frames|/你的/frames路径|g' data/eval_qwenvl.jsonl
#
#    data/eval_vora.jsonl 用的是相对路径 (frames/xxx.jpg),
#    确保从 eval_pack/ 目录能访问到 frames/ 文件夹:
#      ln -s /你的/frames路径 frames
#
# 5. 运行:
#      conda activate eval
#      cd eval_pack
#      bash eval/run_eval.sh
#
# 6. 结果会打印表格并保存到 eval/results/*.json
#
# ========== 目录结构 ==========
#
#   eval_pack/
#   ├── eval/                <- 评测脚本
#   ├── models/              <- VoRA 模型代码
#   ├── tools/               <- merge_lora.py
#   ├── generation_files/    <- tokenizer/processor
#   ├── data/
#   │   ├── eval_vora.jsonl  <- VoRA 系列用 (相对路径)
#   │   └── eval_qwenvl.jsonl <- Qwen-VL 系列用 (绝对路径)
#   ├── vora_merged_250/     <- VoRA merged (~32G)
#   ├── lora_merged/         <- Qwen-VL+LoRA merged (~7G)
#   ├── circulant_merged/    <- Qwen-VL+Circulant merged (~7G)
#   └── requirements_eval.txt
#
# =============================================================

set -e
cd "$(dirname "$0")/.."

# ---------- 路径配置 (根据你的环境修改) ----------
# 基线模型 (另一台机器上已有)
QWEN_MODEL="/path/to/Qwen2.5-7B-Instruct"
QWEN_VL_MODEL="/path/to/Qwen2.5-VL-3B-Instruct"
AIMV2_PATH="/path/to/aimv2-huge-patch14-448"

# 评测数据
EVAL_DATA_VORA="data/eval_vora.jsonl"       # VoRA 系列用 (相对路径 frames/xxx.jpg)
EVAL_DATA_QWEN="data/eval_qwenvl.jsonl"     # Qwen-VL 系列用 (绝对路径, 需要按目标机器修改)

DTYPE="float16"
RESULTS_DIR="eval/results"

# 已 merge 好的模型
VORA_MERGED="vora_merged_250"
VORA_CIRC_MERGED="/path/to/vora-circulant-merged"
LORA_MERGED="lora_merged"
CIRC_MERGED="circulant_merged"
# -----------------------------------------------

mkdir -p "$RESULTS_DIR"

pip install nltk rouge-score qwen-vl-utils 2>/dev/null || true
python -c "import nltk; nltk.download('punkt', quiet=True); nltk.download('punkt_tab', quiet=True)" 2>/dev/null || true

echo ""
echo "=============================================="
echo "  Step 1: Qwen2.5-7B Text-Only Baseline"
echo "=============================================="
python eval/eval_qwen_baseline.py \
    --mode all \
    --model-path "$QWEN_MODEL" \
    --eval-data "$EVAL_DATA_QWEN" \
    --max-samples 200 \
    --max-new-tokens 256 \
    --dtype "$DTYPE" \
    --output "$RESULTS_DIR/qwen_baseline.json"

echo ""
echo "=============================================="
echo "  Step 2: VoRA Full Eval"
echo "=============================================="
python eval/eval_vora.py \
    --mode all \
    --checkpoint "$VORA_MERGED" \
    --merged \
    --eval-data "$EVAL_DATA_VORA" \
    --image-processor "$AIMV2_PATH" \
    --max-samples 200 \
    --max-new-tokens 256 \
    --dtype "$DTYPE" \
    --output "$RESULTS_DIR/vora_best.json"

echo ""
echo "=============================================="
echo "  Step 3: VoRA-Circulant Full Eval"
echo "=============================================="
python eval/eval_vora.py \
    --mode all \
    --checkpoint "$VORA_CIRC_MERGED" \
    --merged \
    --eval-data "$EVAL_DATA_VORA" \
    --image-processor "$AIMV2_PATH" \
    --max-samples 200 \
    --max-new-tokens 256 \
    --dtype "$DTYPE" \
    --output "$RESULTS_DIR/vora_circulant.json"

echo ""
echo "=============================================="
echo "  Step 4: Qwen2.5-VL-3B Original Baseline"
echo "=============================================="
python eval/eval_qwen_vl.py \
    --mode all \
    --model-path "$QWEN_VL_MODEL" \
    --eval-data "$EVAL_DATA_QWEN" \
    --max-samples 200 \
    --max-new-tokens 256 \
    --dtype "$DTYPE" \
    --output "$RESULTS_DIR/qwen_vl_original.json"

echo ""
echo "=============================================="
echo "  Step 5: Qwen2.5-VL-3B + LoRA"
echo "=============================================="
if [ -d "$LORA_MERGED" ]; then
    python eval/eval_qwen_vl.py \
        --mode all \
        --model-path "$LORA_MERGED" \
        --eval-data "$EVAL_DATA_QWEN" \
        --max-samples 200 \
        --max-new-tokens 256 \
        --dtype "$DTYPE" \
        --output "$RESULTS_DIR/qwen_vl_lora.json"
else
    echo "LoRA merged model not found at $LORA_MERGED, skipping."
fi

echo ""
echo "=============================================="
echo "  Step 6: Qwen2.5-VL-3B + Block-Circulant"
echo "=============================================="
if [ -d "$CIRC_MERGED" ]; then
    python eval/eval_qwen_vl.py \
        --mode all \
        --model-path "$CIRC_MERGED" \
        --eval-data "$EVAL_DATA_QWEN" \
        --max-samples 200 \
        --max-new-tokens 256 \
        --dtype "$DTYPE" \
        --output "$RESULTS_DIR/qwen_vl_circulant.json"
else
    echo "Block-Circulant merged model not found at $CIRC_MERGED, skipping."
fi

echo ""
echo "=============================================="
echo "  Summary"
echo "=============================================="
python -c "
import json, os, math

results_dir = '$RESULTS_DIR'

print('='*70)
print(f'{\"Model\":<35} {\"Loss\":>10} {\"PPL\":>8} {\"BLEU-4\":>8} {\"ROUGE-L\":>8}')
print('-'*70)

def print_row(name, filepath):
    if not os.path.exists(filepath):
        return
    r = json.load(open(filepath))
    ppl = r.get('perplexity', None)
    loss = math.log(ppl) if ppl and ppl != float('inf') else None
    b4 = r.get('BLEU-4', None)
    rl = r.get('ROUGE-L', None)
    loss_s = f'{loss:.4f}' if loss else '-'
    ppl_s = f'{ppl:.2f}' if ppl else '-'
    b4_s = f'{b4:.4f}' if b4 else '-'
    rl_s = f'{rl:.4f}' if rl else '-'
    print(f'{name:<35} {loss_s:>10} {ppl_s:>8} {b4_s:>8} {rl_s:>8}')

print_row('Qwen2.5-7B (text-only)', os.path.join(results_dir, 'qwen_baseline.json'))
print_row('VoRA', os.path.join(results_dir, 'vora_best.json'))
print_row('VoRA-Circulant', os.path.join(results_dir, 'vora_circulant.json'))

print('-'*70)

print_row('Qwen2.5-VL-3B (original)', os.path.join(results_dir, 'qwen_vl_original.json'))
print_row('Qwen2.5-VL-3B + LoRA', os.path.join(results_dir, 'qwen_vl_lora.json'))
print_row('Qwen2.5-VL-3B + Circulant', os.path.join(results_dir, 'qwen_vl_circulant.json'))

print('='*70)
"

echo ""
echo "All results saved to \$RESULTS_DIR/"
echo "Done!"