#!/bin/bash # ============================================================= # VoRA Evaluation Runner (打包版) # ============================================================= # # ========== 使用教程 ========== # # 1. 把 eval_pack/ 整个文件夹传到目标机器 # # 2. 创建 conda 环境 (一次性): # conda create -n eval python=3.10 -y # conda activate eval # cd eval_pack # pip install -r requirements_eval.txt # # 3. 修改下方「路径配置」部分: # - QWEN_MODEL: Qwen2.5-7B-Instruct 路径 # - QWEN_VL_MODEL: Qwen2.5-VL-3B-Instruct 路径 # - AIMV2_PATH: aimv2-huge-patch14-448 路径 # - VORA_CIRC_MERGED: VoRA-Circulant merged 模型路径 # # 4. 修改 data/eval_qwenvl.jsonl 中的图片绝对路径: # sed -i 's|/share/home/jcdl1lsy2clx/VoRA/data_dir/VoRA-Recap-29M/frames|/你的/frames路径|g' data/eval_qwenvl.jsonl # # data/eval_vora.jsonl 用的是相对路径 (frames/xxx.jpg), # 确保从 eval_pack/ 目录能访问到 frames/ 文件夹: # ln -s /你的/frames路径 frames # # 5. 运行: # conda activate eval # cd eval_pack # bash eval/run_eval.sh # # 6. 结果会打印表格并保存到 eval/results/*.json # # ========== 目录结构 ========== # # eval_pack/ # ├── eval/ <- 评测脚本 # ├── models/ <- VoRA 模型代码 # ├── tools/ <- merge_lora.py # ├── generation_files/ <- tokenizer/processor # ├── data/ # │ ├── eval_vora.jsonl <- VoRA 系列用 (相对路径) # │ └── eval_qwenvl.jsonl <- Qwen-VL 系列用 (绝对路径) # ├── vora_merged_250/ <- VoRA merged (~32G) # ├── lora_merged/ <- Qwen-VL+LoRA merged (~7G) # ├── circulant_merged/ <- Qwen-VL+Circulant merged (~7G) # └── requirements_eval.txt # # ============================================================= set -e cd "$(dirname "$0")/.." # ---------- 路径配置 (根据你的环境修改) ---------- # 基线模型 (另一台机器上已有) QWEN_MODEL="/path/to/Qwen2.5-7B-Instruct" QWEN_VL_MODEL="/path/to/Qwen2.5-VL-3B-Instruct" AIMV2_PATH="/path/to/aimv2-huge-patch14-448" # 评测数据 EVAL_DATA_VORA="data/eval_vora.jsonl" # VoRA 系列用 (相对路径 frames/xxx.jpg) EVAL_DATA_QWEN="data/eval_qwenvl.jsonl" # Qwen-VL 系列用 (绝对路径, 需要按目标机器修改) DTYPE="float16" RESULTS_DIR="eval/results" # 已 merge 好的模型 VORA_MERGED="vora_merged_250" VORA_CIRC_MERGED="/path/to/vora-circulant-merged" LORA_MERGED="lora_merged" CIRC_MERGED="circulant_merged" # ----------------------------------------------- mkdir -p "$RESULTS_DIR" pip install nltk rouge-score qwen-vl-utils 2>/dev/null || true python -c "import nltk; nltk.download('punkt', quiet=True); nltk.download('punkt_tab', quiet=True)" 2>/dev/null || true echo "" echo "==============================================" echo " Step 1: Qwen2.5-7B Text-Only Baseline" echo "==============================================" python eval/eval_qwen_baseline.py \ --mode all \ --model-path "$QWEN_MODEL" \ --eval-data "$EVAL_DATA_QWEN" \ --max-samples 200 \ --max-new-tokens 256 \ --dtype "$DTYPE" \ --output "$RESULTS_DIR/qwen_baseline.json" echo "" echo "==============================================" echo " Step 2: VoRA Full Eval" echo "==============================================" python eval/eval_vora.py \ --mode all \ --checkpoint "$VORA_MERGED" \ --merged \ --eval-data "$EVAL_DATA_VORA" \ --image-processor "$AIMV2_PATH" \ --max-samples 200 \ --max-new-tokens 256 \ --dtype "$DTYPE" \ --output "$RESULTS_DIR/vora_best.json" echo "" echo "==============================================" echo " Step 3: VoRA-Circulant Full Eval" echo "==============================================" python eval/eval_vora.py \ --mode all \ --checkpoint "$VORA_CIRC_MERGED" \ --merged \ --eval-data "$EVAL_DATA_VORA" \ --image-processor "$AIMV2_PATH" \ --max-samples 200 \ --max-new-tokens 256 \ --dtype "$DTYPE" \ --output "$RESULTS_DIR/vora_circulant.json" echo "" echo "==============================================" echo " Step 4: Qwen2.5-VL-3B Original Baseline" echo "==============================================" python eval/eval_qwen_vl.py \ --mode all \ --model-path "$QWEN_VL_MODEL" \ --eval-data "$EVAL_DATA_QWEN" \ --max-samples 200 \ --max-new-tokens 256 \ --dtype "$DTYPE" \ --output "$RESULTS_DIR/qwen_vl_original.json" echo "" echo "==============================================" echo " Step 5: Qwen2.5-VL-3B + LoRA" echo "==============================================" if [ -d "$LORA_MERGED" ]; then python eval/eval_qwen_vl.py \ --mode all \ --model-path "$LORA_MERGED" \ --eval-data "$EVAL_DATA_QWEN" \ --max-samples 200 \ --max-new-tokens 256 \ --dtype "$DTYPE" \ --output "$RESULTS_DIR/qwen_vl_lora.json" else echo "LoRA merged model not found at $LORA_MERGED, skipping." fi echo "" echo "==============================================" echo " Step 6: Qwen2.5-VL-3B + Block-Circulant" echo "==============================================" if [ -d "$CIRC_MERGED" ]; then python eval/eval_qwen_vl.py \ --mode all \ --model-path "$CIRC_MERGED" \ --eval-data "$EVAL_DATA_QWEN" \ --max-samples 200 \ --max-new-tokens 256 \ --dtype "$DTYPE" \ --output "$RESULTS_DIR/qwen_vl_circulant.json" else echo "Block-Circulant merged model not found at $CIRC_MERGED, skipping." fi echo "" echo "==============================================" echo " Summary" echo "==============================================" python -c " import json, os, math results_dir = '$RESULTS_DIR' print('='*70) print(f'{\"Model\":<35} {\"Loss\":>10} {\"PPL\":>8} {\"BLEU-4\":>8} {\"ROUGE-L\":>8}') print('-'*70) def print_row(name, filepath): if not os.path.exists(filepath): return r = json.load(open(filepath)) ppl = r.get('perplexity', None) loss = math.log(ppl) if ppl and ppl != float('inf') else None b4 = r.get('BLEU-4', None) rl = r.get('ROUGE-L', None) loss_s = f'{loss:.4f}' if loss else '-' ppl_s = f'{ppl:.2f}' if ppl else '-' b4_s = f'{b4:.4f}' if b4 else '-' rl_s = f'{rl:.4f}' if rl else '-' print(f'{name:<35} {loss_s:>10} {ppl_s:>8} {b4_s:>8} {rl_s:>8}') print_row('Qwen2.5-7B (text-only)', os.path.join(results_dir, 'qwen_baseline.json')) print_row('VoRA', os.path.join(results_dir, 'vora_best.json')) print_row('VoRA-Circulant', os.path.join(results_dir, 'vora_circulant.json')) print('-'*70) print_row('Qwen2.5-VL-3B (original)', os.path.join(results_dir, 'qwen_vl_original.json')) print_row('Qwen2.5-VL-3B + LoRA', os.path.join(results_dir, 'qwen_vl_lora.json')) print_row('Qwen2.5-VL-3B + Circulant', os.path.join(results_dir, 'qwen_vl_circulant.json')) print('='*70) " echo "" echo "All results saved to \$RESULTS_DIR/" echo "Done!"