# conda activate VisRAG # To-Do List ======================================================================== 1. Baseline: clear model to clear, degra, real datasets (Ret -> Gen) 2. Training Code: dataloader, synthetic function # Environments ========================================================================= git clone https://github.com/OpenBMB/VisRAG.git conda create --name VisRAG python==3.10.8 conda activate VisRAG conda install nvidia/label/cuda-11.8.0::cuda-toolkit cd VisRAG pip install -r requirements.txt pip install -e . cd timm_modified pip install -e . pip skimage pip install peft=0.7.1 cd .. % Download & Change cache_dir /home/work/shared-fi-datasets-01/users/hsiang.chen/Project/Robust/VisRAG/src/openmatch/arguments.py L136. % soft link pre-trained models ln -s /home/work/shared-fi-datasets-01/users/hsiang.chen/Project/Robust/ModelZoo/* /home/work/shared-fi-datasets-01/users/hsiang.chen/Project/Robust/VisRAG/checkpoints # Dataset format ========================================================================= qrels: ['query_id', 'corpus_id', 'score'] * clear: (queries, qrels, corpus) openbmb/VisRAG-Ret-Test-PlotQA openbmb/VisRAG-Ret-Test-SlideVQA openbmb/VisRAG-Ret-Test-InfoVQA openbmb/VisRAG-Ret-Test-ArxivQA openbmb/VisRAG-Ret-Test-ChartQA openbmb/VisRAG-Ret-Test-MP-DocVQA * synthetic-degradation: (corpus) -> !!! please use clear queries and qrels openbmb/VisRAG-Ret-Test-PlotQA openbmb/VisRAG-Ret-Test-SlideVQA openbmb/VisRAG-Ret-Test-InfoVQA openbmb/VisRAG-Ret-Test-ArxivQA openbmb/VisRAG-Ret-Test-ChartQA openbmb/VisRAG-Ret-Test-MP-DocVQA * real-world: (queries, qrels, corpus) rweics5cs7/exo7-realworld-db-combined (RVL-CDIP-clear) rweics5cs7/exo7-realworld-db-combined-deg (RVL-CDIP-degra) rweics5cs7/exo7-realworld-db-combined-deg-fixed (RVL-CDIP-filter) rweics5cs7/exo9-realworld-db-combined (MP-DocVQA) rweics5cs7/exo10-realworld-db-combined (ArxivQA) ========================================================================= # Training ========================================================================= 1. Template: # MAX_SEQ_LEN, PER_DEV_BATCH_SIZE, GPUS_PER_NODE, SOFTMAX_TEMPERATURE, EPOCH, QUERY_INSTRUCTION, CORPUS_INSTRUCTION, DEEPSPEED, LR, MAPPING, POOLING, ATTENTION, NPASSAGE, GRADCACHE, GRADCACHE_MICRO, PASSAGE_STOP_GRAD, MODEL_PATH, DATASET_PATH, SYNTHETIC_DISTORTION TAG LoRA bash scripts/train_retriever/train.sh 2048 32 8 0.02 5 true false config/deepspeed.json 1e-5 true wmean causal 1 true 4 false ./checkpoints/openbmb/VisRAG-Ret all true Robust false bash scripts/train_retriever/train.sh 2048 32 8 0.02 2 true false config/deepspeed.json 1e-5 true wmean causal 1 true 4 false /home/work/shared-fi-datasets-01/users/hsiang.chen/Project/Robust/VisRAG/checkpoints/Robust-train-2025-11-09-181056-lr-1e-5-temp-0.02-bsz32-ngpus8-nnodes1-inbatch--nepoch-3-pooling-wmean-attention-causal-qinstruct-true-cinstruct-false-gradcache-true-passage-stopgrad-false-npassage-1/checkpoint-2500 all true RobustRet false ## Configuration ### Model: source: /home/work/shared-fi-datasets-01/users/hsiang.chen/Project/Robust/ModelZoo/ "SigLIP": "google/SigLIP", "CPM-2B", "VisRAG": "openbmb/VisRAG-Ret" ### Data: (try real) 1. Datasets: "out_domain", "in_domain", "real", "all", "openbmb/VisRAG-Ret-Train-In-domain-data", "openbmb/VisRAG-Ret-Train-Synthetic-data" or any huggingface data. 2. Loader: MAPPING=True, use_mapping_dataset -> MappingMMDRTrainDataset (not use StreamMMDRTrainDataset (only support single dataset)) * columns = [image, source, query] #### -> If you wish to train using your own datasets, remove the `--from_hf_repo` line from the `train.sh` script. Additionally, ensure that your dataset directory contains a `metadata.json` file, which must include a `length` field specifying the total number of samples in the dataset. * src/openmatch/driver/train.py # Evaluation ========================================================================= ## Ret: ==================== * source /home/work/shared-fi-datasets-01/users/hsiang.chen/.bashrc && conda activate VisRAG && cd /home/work/shared-fi-datasets-01/users/hsiang.chen/Project/Robust/VisRAG ### Testing for baseline 1. (Clear) bash scripts/eval_retriever/eval.sh 512 2048 16 1 wmean causal ArxivQA,ChartQA,MP-DocVQA,InfoVQA,PlotQA,SlideVQA /home/work/shared-fi-datasets-01/users/hsiang.chen/Project/Robust/ModelZoo/openbmb/VisRAG-Ret 0 ./results/retrieval_clear false 2. (Degra) bash scripts/eval_retriever/eval_degra.sh 512 2048 16 1 wmean causal ArxivQA,ChartQA,MP-DocVQA,InfoVQA,PlotQA,SlideVQA /home/work/shared-fi-datasets-01/users/hsiang.chen/Project/Robust/ModelZoo/openbmb/VisRAG-Ret 0 ./results/retrieval_clear false 2. (Real-World) * exo7-realworld-db-combined-deg (rvl-cdip), exo9-realworld-db-combined (MP-DocVQA), exo10-realworld-db-combined (ArxivQA) bash scripts/eval_retriever/eval_real.sh 512 2048 16 1 wmean causal exo7-realworld-db-combined-deg,exo9-realworld-db-combined,exo10-realworld-db-combined /home/work/shared-fi-datasets-01/users/hsiang.chen/Project/Robust/ModelZoo/openbmb/VisRAG-Ret 0 ./results/retrieval_clear false rweics5cs7/exo7-realworld-db-combined-deg-fixed bash scripts/eval_retriever/eval_real.sh 512 2048 16 4 wmean causal exo7-realworld-db-combined-deg /home/work/shared-fi-datasets-01/users/hsiang.chen/Project/Robust/ModelZoo/openbmb/VisRAG-Ret 0 ./results/retrieval_clear false bash scripts/eval_retriever/eval_real.sh 512 2048 16 4 wmean causal exo7-realworld-db-combined-deg /home/work/shared-fi-datasets-01/users/hsiang.chen/Project/Robust/VisRAG/checkpoints/fft_syn 0 ./results/retrieval_fft false bash scripts/eval_retriever/eval_real.sh 512 2048 16 4 wmean causal exo7-realworld-db-combined-deg /home/work/shared-fi-datasets-01/users/hsiang.chen/Project/Robust/VisRAG/checkpoints/lora 0 ./results/retrieval_lora true ### Testing for SFT 1. (clear) bash scripts/eval_retriever/eval.sh 512 2048 16 1 wmean causal ArxivQA,ChartQA,MP-DocVQA,InfoVQA,PlotQA,SlideVQA /home/work/shared-fi-datasets-01/users/hsiang.chen/Project/Robust/VisRAG/checkpoints/fft_syn 0 ./results/retrieval_fft false 2. (degra) bash scripts/eval_retriever/eval_degra.sh 512 2048 16 1 wmean causal ArxivQA,ChartQA,MP-DocVQA,InfoVQA,PlotQA,SlideVQA /home/work/shared-fi-datasets-01/users/hsiang.chen/Project/Robust/VisRAG/checkpoints/fft_syn 0 ./results/retrieval_fft false 3. (real-world) bash scripts/eval_retriever/eval_real.sh 512 2048 16 1 wmean causal exo7-realworld-db-combined-deg,exo9-realworld-db-combined,exo10-realworld-db-combined /home/work/shared-fi-datasets-01/users/hsiang.chen/Project/Robust/VisRAG/checkpoints/fft_syn 0 ./results/retrieval_fft false ### Testing for LoRA 1. (clear) bash scripts/eval_retriever/eval.sh 512 2048 16 1 wmean causal ArxivQA,ChartQA,MP-DocVQA,InfoVQA,PlotQA,SlideVQA /home/work/shared-fi-datasets-01/users/hsiang.chen/Project/Robust/VisRAG/checkpoints/lora 0 ./results/retrieval_lora true 2. (degra) bash scripts/eval_retriever/eval_degra.sh 512 2048 16 1 wmean causal ArxivQA,ChartQA,MP-DocVQA,InfoVQA,PlotQA,SlideVQA /home/work/shared-fi-datasets-01/users/hsiang.chen/Project/Robust/VisRAG/checkpoints/lora 0 ./results/retrieval_lora true 3. (real-world) bash scripts/eval_retriever/eval_real.sh 512 2048 16 1 wmean causal exo7-realworld-db-combined-deg,exo9-realworld-db-combined,exo10-realworld-db-combined /home/work/shared-fi-datasets-01/users/hsiang.chen/Project/Robust/VisRAG/checkpoints/lora 0 ./results/retrieval_lora true ### Testing for Ours 1. (clear) bash scripts/eval_retriever/eval.sh 512 2048 16 1 wmean causal ChartQA,ArxivQA,MP-DocVQA,InfoVQA,PlotQA,SlideVQA /home/work/shared-fi-datasets-01/users/hsiang.chen/Project/Robust/VisRAG/checkpoints/Robust-train-2025-11-09-181056-lr-1e-5-temp-0.02-bsz32-ngpus8-nnodes1-inbatch--nepoch-3-pooling-wmean-attention-causal-qinstruct-true-cinstruct-false-gradcache-true-passage-stopgrad-false-npassage-1/checkpoint-2500 0 ./results/retrieval_robust false 2. (degra) bash scripts/eval_retriever/eval_degra.sh 512 2048 16 1 wmean causal ChartQA,ArxivQA,MP-DocVQA,InfoVQA,PlotQA,SlideVQA /home/work/shared-fi-datasets-01/users/hsiang.chen/Project/Robust/VisRAG/checkpoints/Robust-train-2025-11-09-181056-lr-1e-5-temp-0.02-bsz32-ngpus8-nnodes1-inbatch--nepoch-3-pooling-wmean-attention-causal-qinstruct-true-cinstruct-false-gradcache-true-passage-stopgrad-false-npassage-1/checkpoint-2500 0 ./results/retrieval_robust false 3. (real-world) bash scripts/eval_retriever/eval_real.sh 512 2048 16 1 wmean causal exo7-realworld-db-combined-deg,exo9-realworld-db-combined,exo10-realworld-db-combined /home/work/shared-fi-datasets-01/users/hsiang.chen/Project/Robust/VisRAG/checkpoints/Robust-train-2025-11-09-181056-lr-1e-5-temp-0.02-bsz32-ngpus8-nnodes1-inbatch--nepoch-3-pooling-wmean-attention-causal-qinstruct-true-cinstruct-false-gradcache-true-passage-stopgrad-false-npassage-1/checkpoint-2500 0 ./results/retrieval_robust false bash scripts/eval_retriever/eval_real.sh 512 2048 16 1 wmean causal exo7-realworld-db-combined-deg,exo9-realworld-db-combined,exo10-realworld-db-combined /home/work/shared-fi-datasets-01/users/hsiang.chen/Project/Robust/VisRAG/checkpoints/Robust-train-2025-11-09-181056-lr-1e-5-temp-0.02-bsz32-ngpus8-nnodes1-inbatch--nepoch-3-pooling-wmean-attention-causal-qinstruct-true-cinstruct-false-gradcache-true-passage-stopgrad-false-npassage-1/checkpoint-2500 0 ./results/retrieval_robust false -> USE /home/work/shared-fi-datasets-01/users/hsiang.chen/Project/Robust/VisRAG/stastic_ret.py stastic results. ====================== ## Gen: (should run Ret first and get the embedding and reterival results) ====================== 1. (Clear) python scripts/generate/generate.py \ --model_name MiniCPMV2.6 \ --model_name_or_path /home/work/shared-fi-datasets-01/users/hsiang.chen/Project/Robust/ModelZoo/openbmb/MiniCPM-V-2_6 \ --dataset_name ChartQA \ --dataset_name_or_path openbmb/VisRAG-Ret-Test-ChartQA \ --rank 0 \ --world_size 1 \ --topk 3 \ --results_root_dir /home/work/shared-fi-datasets-01/users/hsiang.chen/Project/Robust/VisRAG/data/checkpoints/clear \ --task_type multi_image \ --concatenate_type horizontal \ --output_dir /home/work/shared-fi-datasets-01/users/hsiang.chen/Project/Robust/VisRAG/data/checkpoints/generate * model_name: ['MiniCPM', 'MiniCPMV2.0', 'MiniCPMV2.6', 'gpt4o'] * model_name_or_path: /home/work/shared-fi-datasets-01/users/hsiang.chen/Project/Robust/ModelZoo/openbmb/MiniCPM-V-2_6 * dataset_name: ['ArxivQA', 'ChartQA', 'PlotQA', 'MP-DocVQA', 'SlideVQA', 'InfoVQA'] * dataset_name_or_path: * task_type: ['text', 'page_concatenation', 'weighted_selection', 'multi_image'] * concatenate: ['horizontal', 'vertical'] 2. (Ret) python scripts/generate/generate.py \ --model_name MiniCPMV2.6 \ --model_name_or_path /home/work/shared-fi-datasets-01/users/hsiang.chen/Project/Robust/ModelZoo/openbmb/MiniCPM-V-2_6 \ --dataset_name ChartQA \ --dataset_name_or_path openbmb/VisRAG-Ret-Test-ChartQA \ --rank 0 \ --world_size 1 \ --topk 3 \ --results_root_dir /home/work/shared-fi-datasets-01/users/hsiang.chen/Project/Robust/VisRAG/data/checkpoints/clear \ --task_type multi_image \ --concatenate_type horizontal \ --output_dir /home/work/shared-fi-datasets-01/users/hsiang.chen/Project/Robust/VisRAG/data/checkpoints/generate # config ==================================================================== Evaluating: ArxivQA This dataset result dir: /data/checkpoints/eval-2025-09-28-182703-maxq-512-maxp-2048-bsz-16-pooling-wmean-attention-causal-gpus-per-node-1/ArxivQA CORPUS_PATH: openbmb/VisRAG-Ret-Test-ArxivQA QUERY_PATH: openbmb/VisRAG-Ret-Test-ArxivQA QRELS_PATH: openbmb/VisRAG-Ret-Test-ArxivQA ==================================================================================================== model_args: ModelArguments(model_name_or_path='/home/work/shared-fi-datasets-01/users/hsiang.chen/Project/Robust/ModelZoo/VisRAG-Ret', target_model_path=None, config_name=None, tokenizer_name=None, processor_name=None, cache_dir='/home/work/shared-fi-datasets-01/users/hsiang.chen/Project/Robust/ModelZoo', untie_encoder=False, feature='last_hidden_state', pooling='wmean', attention='causal', add_linear_head=False, projection_in_dim=768, projection_out_dim=768, dtype='float16', encoder_only=False, pos_token=None, neg_token=None, normalize=True, lora=False, lora_r=32, attn_implementation='sdpa') data_args: DataArguments(train_dir=None, train_path=None, eval_path=None, qrels_path='openbmb/VisRAG-Ret-Test-MP-DocVQA', query_path='openbmb/VisRAG-Ret-Test-MP-DocVQA', corpus_path='openbmb/VisRAG-Ret-Test-MP-DocVQA', data_dir=None, train_n_passages=8, positive_passage_no_shuffle=False, negative_passage_no_shuffle=False, q_max_len=512, p_max_len=2048, query_instruction=True, corpus_instruction=False, data_cache_dir='/home/work/shared-fi-datasets-01/users/hsiang.chen/Project/Robust/Dataset', query_template='Represent this query for retrieving relevant documents: ', query_column_names=None, doc_template='', doc_column_names=None, all_markers=None, encode_as_text_pair=False, from_hf_repo=True)