evaluation/1ours_eval_aio.sh

echo '========================================================================================'
echo 'This script is for evaluating the results of our method on the test sets.'
echo 'Test sets include union_test, 613, and genre test sets, see our_eval_xxx.sh for details.'
echo 'This script will execute the following steps:'
echo '1. Box filter'
echo '2. Filter prediction within test set'
echo '3. Generate evaluation result'
echo '4. Eval2latex'
echo '5. Context eval'
echo 'Don''t forget to set ITB_PRED, SEM_PRED, and EVAL in this script.'
echo '========================================================================================'

# Don't forget this
# VAR GPT4V
# export ITB_PRED=aaa_icse_test_set_merged_gpt4v_gpt4v_ape_d_object_bbox_r1i
# export SEM_PRED=aaa_icse_test_set_merged_gpt4v_gpt4v_ape_d_object_bbox_r1s
# VAR GPT4o
# export ITB_PRED=realfse_union3_gpt4v_ape_d_object_bbox_gpu0123_i
# export SEM_PRED=realfse_union3_gpt4v_ape_d_object_bbox_gpu0123_s
# VAR CLAUDE
# export ITB_PRED=realfse_union3_claude35sonnet_ape_d_object_bbox_gpu0123_i
# export SEM_PRED=realfse_union3_claude35sonnet_ape_d_object_bbox_gpu0123_s
# VAR GEMINI
export ITB_PRED=realfse_union3_gemini15pro_ape_d_object_bbox_gpu0123_i
export SEM_PRED=realfse_union3_gemini15pro_ape_d_object_bbox_gpu0123_s
# ABLATION 1
# export ITB_PRED=without_module_1_i
# export SEM_PRED=without_module_1_s
# ABLATION 2
# export ITB_PRED=without_module_2_i
# export SEM_PRED=without_module_2_s
# ABLATION 3
# export ITB_PRED=without_module_3_i
# export SEM_PRED=without_module_3_s

# export EVAL=normal
export EVAL=sample


echo "Current ITB_PRED = ${ITB_PRED}"
echo "Current SEM_PRED = ${SEM_PRED}"
echo "Current EVAL = ${EVAL}"

# Mandatory step, DO NOT OMIT
echo ">>> box filter >>>"
cp ./results/ours/${ITB_PRED}.json ./results/ours/${ITB_PRED}_bf.json
cp ./results/ours/${SEM_PRED}.json ./results/ours/${SEM_PRED}_bf.json
export ITB_PRED=${ITB_PRED}_bf
export SEM_PRED=${SEM_PRED}_bf
python filter_box.py -i ./results/ours/${ITB_PRED}.json -o ./results/ours/${ITB_PRED}.json -n -hu
python filter_box.py -i ./results/ours/${SEM_PRED}.json -o ./results/ours/${SEM_PRED}.json -n -hu -c
echo "<<< box filter <<<"


echo ">>> filter prediction within test set >>>"
bash filter_test_prediction.sh
echo "<<< filter prediction within test set <<<"


echo ">>> generate evaluation result >>>"
mkdir -p ./eval_results/ours
bash our_eval_${EVAL}.sh
echo "<<< generate evaluation result <<<"


# echo ">>> eval2latex >>>"
mkdir -p ./eval_latex/ours
python eval_latex/eval2latex_performance_comparison.py \
    -ia ./eval_results/ours/${ITB_PRED}_613_i.csv \
    -sa ./eval_results/ours/${SEM_PRED}_613_s.csv \
    -ig ./eval_results/ours/${ITB_PRED}_genre_i.csv \
    -sg ./eval_results/ours/${SEM_PRED}_genre_s.csv \
    -o ./eval_latex/ours/${SEM_PRED}.tex
# echo "<<< eval2latex <<<"


# echo ">>> context eval >>>"
# bash our_context_eval.sh
# echo "<<< context eval <<<"


echo 'Done.'