Upload Project/CIH.md with huggingface_hub

Project/CIH.md

@@ -0,0 +1,180 @@
+# conda activate VisRAG
+
+# To-Do List
+========================================================================
+1. Baseline: clear model to clear, degra, real datasets (Ret -> Gen)
+2. Training Code: dataloader, synthetic function
+
+# Environments
+=========================================================================
+git clone https://github.com/OpenBMB/VisRAG.git
+conda create --name VisRAG python==3.10.8
+conda activate VisRAG
+conda install nvidia/label/cuda-11.8.0::cuda-toolkit
+cd VisRAG
+pip install -r requirements.txt
+pip install -e .
+cd timm_modified
+pip install -e .
+pip skimage
+pip install peft=0.7.1
+cd ..
+% Download & Change cache_dir
+/home/work/shared-fi-datasets-01/users/hsiang.chen/Project/Robust/VisRAG/src/openmatch/arguments.py L136.
+% soft link pre-trained models
+ln -s /home/work/shared-fi-datasets-01/users/hsiang.chen/Project/Robust/ModelZoo/* /home/work/shared-fi-datasets-01/users/hsiang.chen/Project/Robust/VisRAG/checkpoints
+
+# Dataset format
+========================================================================= 
+qrels: ['query_id', 'corpus_id', 'score']
+* clear: (queries, qrels, corpus)
+openbmb/VisRAG-Ret-Test-PlotQA 
+openbmb/VisRAG-Ret-Test-SlideVQA 
+openbmb/VisRAG-Ret-Test-InfoVQA 
+openbmb/VisRAG-Ret-Test-ArxivQA 
+openbmb/VisRAG-Ret-Test-ChartQA 
+openbmb/VisRAG-Ret-Test-MP-DocVQA
+
+* synthetic-degradation: (corpus) -> !!! please use clear queries and qrels
+openbmb/VisRAG-Ret-Test-PlotQA 
+openbmb/VisRAG-Ret-Test-SlideVQA 
+openbmb/VisRAG-Ret-Test-InfoVQA 
+openbmb/VisRAG-Ret-Test-ArxivQA 
+openbmb/VisRAG-Ret-Test-ChartQA 
+openbmb/VisRAG-Ret-Test-MP-DocVQA
+
+* real-world: (queries, qrels, corpus)
+rweics5cs7/exo7-realworld-db-combined (RVL-CDIP-clear) 
+rweics5cs7/exo7-realworld-db-combined-deg (RVL-CDIP-degra) 
+rweics5cs7/exo7-realworld-db-combined-deg-fixed (RVL-CDIP-filter) 
+rweics5cs7/exo9-realworld-db-combined (MP-DocVQA) 
+rweics5cs7/exo10-realworld-db-combined (ArxivQA) 
+========================================================================= 
+
+# Training
+========================================================================= 
+1. Template:
+    # MAX_SEQ_LEN, PER_DEV_BATCH_SIZE, GPUS_PER_NODE, SOFTMAX_TEMPERATURE, EPOCH, QUERY_INSTRUCTION, CORPUS_INSTRUCTION, DEEPSPEED, LR, MAPPING, POOLING, ATTENTION, NPASSAGE, GRADCACHE, GRADCACHE_MICRO, PASSAGE_STOP_GRAD, MODEL_PATH, DATASET_PATH, SYNTHETIC_DISTORTION TAG LoRA
+    bash scripts/train_retriever/train.sh 2048 32 8 0.02 5 true false config/deepspeed.json 1e-5 true wmean causal 1 true 4 false ./checkpoints/openbmb/VisRAG-Ret all true Robust false
+
+    bash scripts/train_retriever/train.sh 2048 32 8 0.02 2 true false config/deepspeed.json 1e-5 true wmean causal 1 true 4 false /home/work/shared-fi-datasets-01/users/hsiang.chen/Project/Robust/VisRAG/checkpoints/Robust-train-2025-11-09-181056-lr-1e-5-temp-0.02-bsz32-ngpus8-nnodes1-inbatch--nepoch-3-pooling-wmean-attention-causal-qinstruct-true-cinstruct-false-gradcache-true-passage-stopgrad-false-npassage-1/checkpoint-2500 all true RobustRet false
+
+
+    ## Configuration
+    ### Model: 
+        source: /home/work/shared-fi-datasets-01/users/hsiang.chen/Project/Robust/ModelZoo/
+        "SigLIP": "google/SigLIP", "CPM-2B", "VisRAG": "openbmb/VisRAG-Ret"
+    ### Data: (try real)
+        1. Datasets: "out_domain", "in_domain", "real", "all", "openbmb/VisRAG-Ret-Train-In-domain-data", "openbmb/VisRAG-Ret-Train-Synthetic-data" or any huggingface data.
+        2. Loader: MAPPING=True, use_mapping_dataset -> MappingMMDRTrainDataset  (not use StreamMMDRTrainDataset (only support single dataset))
+            * columns = [image, source, query]
+        #### -> If you wish to train using your own datasets, remove the `--from_hf_repo` line from the `train.sh` script. Additionally, ensure that your dataset directory contains a `metadata.json` file, which must include a `length` field specifying the total number of samples in the dataset.
+
+    * src/openmatch/driver/train.py
+
+
+# Evaluation
+=========================================================================
+## Ret: 
+====================
+* source /home/work/shared-fi-datasets-01/users/hsiang.chen/.bashrc && conda activate VisRAG && cd /home/work/shared-fi-datasets-01/users/hsiang.chen/Project/Robust/VisRAG
+### Testing for baseline
+1. (Clear)
+    bash scripts/eval_retriever/eval.sh 512 2048 16 1 wmean causal ArxivQA,ChartQA,MP-DocVQA,InfoVQA,PlotQA,SlideVQA /home/work/shared-fi-datasets-01/users/hsiang.chen/Project/Robust/ModelZoo/openbmb/VisRAG-Ret 0 ./results/retrieval_clear false
+2. (Degra)
+    bash scripts/eval_retriever/eval_degra.sh 512 2048 16 1 wmean causal ArxivQA,ChartQA,MP-DocVQA,InfoVQA,PlotQA,SlideVQA /home/work/shared-fi-datasets-01/users/hsiang.chen/Project/Robust/ModelZoo/openbmb/VisRAG-Ret 0 ./results/retrieval_clear false
+2. (Real-World)
+    * exo7-realworld-db-combined-deg (rvl-cdip), exo9-realworld-db-combined (MP-DocVQA), exo10-realworld-db-combined (ArxivQA)
+    bash scripts/eval_retriever/eval_real.sh 512 2048 16 1 wmean causal exo7-realworld-db-combined-deg,exo9-realworld-db-combined,exo10-realworld-db-combined /home/work/shared-fi-datasets-01/users/hsiang.chen/Project/Robust/ModelZoo/openbmb/VisRAG-Ret 0 ./results/retrieval_clear false
+
+rweics5cs7/exo7-realworld-db-combined-deg-fixed
+
+bash scripts/eval_retriever/eval_real.sh 512 2048 16 4 wmean causal exo7-realworld-db-combined-deg /home/work/shared-fi-datasets-01/users/hsiang.chen/Project/Robust/ModelZoo/openbmb/VisRAG-Ret 0 ./results/retrieval_clear false 
+
+bash scripts/eval_retriever/eval_real.sh 512 2048 16 4 wmean causal exo7-realworld-db-combined-deg /home/work/shared-fi-datasets-01/users/hsiang.chen/Project/Robust/VisRAG/checkpoints/fft_syn 0 ./results/retrieval_fft false 
+
+bash scripts/eval_retriever/eval_real.sh 512 2048 16 4 wmean causal exo7-realworld-db-combined-deg /home/work/shared-fi-datasets-01/users/hsiang.chen/Project/Robust/VisRAG/checkpoints/lora 0 ./results/retrieval_lora true
+
+### Testing for SFT
+1. (clear)
+    bash scripts/eval_retriever/eval.sh 512 2048 16 1 wmean causal ArxivQA,ChartQA,MP-DocVQA,InfoVQA,PlotQA,SlideVQA /home/work/shared-fi-datasets-01/users/hsiang.chen/Project/Robust/VisRAG/checkpoints/fft_syn 0 ./results/retrieval_fft false
+2. (degra)
+    bash scripts/eval_retriever/eval_degra.sh 512 2048 16 1 wmean causal ArxivQA,ChartQA,MP-DocVQA,InfoVQA,PlotQA,SlideVQA /home/work/shared-fi-datasets-01/users/hsiang.chen/Project/Robust/VisRAG/checkpoints/fft_syn 0 ./results/retrieval_fft false
+3. (real-world)
+    bash scripts/eval_retriever/eval_real.sh 512 2048 16 1 wmean causal exo7-realworld-db-combined-deg,exo9-realworld-db-combined,exo10-realworld-db-combined /home/work/shared-fi-datasets-01/users/hsiang.chen/Project/Robust/VisRAG/checkpoints/fft_syn 0 ./results/retrieval_fft false
+
+
+
+### Testing for LoRA
+1. (clear)
+    bash scripts/eval_retriever/eval.sh 512 2048 16 1 wmean causal ArxivQA,ChartQA,MP-DocVQA,InfoVQA,PlotQA,SlideVQA /home/work/shared-fi-datasets-01/users/hsiang.chen/Project/Robust/VisRAG/checkpoints/lora 0 ./results/retrieval_lora true
+2. (degra)
+    bash scripts/eval_retriever/eval_degra.sh 512 2048 16 1 wmean causal ArxivQA,ChartQA,MP-DocVQA,InfoVQA,PlotQA,SlideVQA /home/work/shared-fi-datasets-01/users/hsiang.chen/Project/Robust/VisRAG/checkpoints/lora 0 ./results/retrieval_lora true
+3. (real-world)
+    bash scripts/eval_retriever/eval_real.sh 512 2048 16 1 wmean causal exo7-realworld-db-combined-deg,exo9-realworld-db-combined,exo10-realworld-db-combined /home/work/shared-fi-datasets-01/users/hsiang.chen/Project/Robust/VisRAG/checkpoints/lora 0 ./results/retrieval_lora true
+
+
+### Testing for Ours
+1. (clear)
+    bash scripts/eval_retriever/eval.sh 512 2048 16 1 wmean causal ChartQA,ArxivQA,MP-DocVQA,InfoVQA,PlotQA,SlideVQA /home/work/shared-fi-datasets-01/users/hsiang.chen/Project/Robust/VisRAG/checkpoints/Robust-train-2025-11-09-181056-lr-1e-5-temp-0.02-bsz32-ngpus8-nnodes1-inbatch--nepoch-3-pooling-wmean-attention-causal-qinstruct-true-cinstruct-false-gradcache-true-passage-stopgrad-false-npassage-1/checkpoint-2500 0 ./results/retrieval_robust false
+2. (degra)
+    bash scripts/eval_retriever/eval_degra.sh 512 2048 16 1 wmean causal ChartQA,ArxivQA,MP-DocVQA,InfoVQA,PlotQA,SlideVQA /home/work/shared-fi-datasets-01/users/hsiang.chen/Project/Robust/VisRAG/checkpoints/Robust-train-2025-11-09-181056-lr-1e-5-temp-0.02-bsz32-ngpus8-nnodes1-inbatch--nepoch-3-pooling-wmean-attention-causal-qinstruct-true-cinstruct-false-gradcache-true-passage-stopgrad-false-npassage-1/checkpoint-2500 0 ./results/retrieval_robust false
+3. (real-world)
+    bash scripts/eval_retriever/eval_real.sh 512 2048 16 1 wmean causal exo7-realworld-db-combined-deg,exo9-realworld-db-combined,exo10-realworld-db-combined /home/work/shared-fi-datasets-01/users/hsiang.chen/Project/Robust/VisRAG/checkpoints/Robust-train-2025-11-09-181056-lr-1e-5-temp-0.02-bsz32-ngpus8-nnodes1-inbatch--nepoch-3-pooling-wmean-attention-causal-qinstruct-true-cinstruct-false-gradcache-true-passage-stopgrad-false-npassage-1/checkpoint-2500 0 ./results/retrieval_robust false
+
+    bash scripts/eval_retriever/eval_real.sh 512 2048 16 1 wmean causal exo7-realworld-db-combined-deg,exo9-realworld-db-combined,exo10-realworld-db-combined /home/work/shared-fi-datasets-01/users/hsiang.chen/Project/Robust/VisRAG/checkpoints/Robust-train-2025-11-09-181056-lr-1e-5-temp-0.02-bsz32-ngpus8-nnodes1-inbatch--nepoch-3-pooling-wmean-attention-causal-qinstruct-true-cinstruct-false-gradcache-true-passage-stopgrad-false-npassage-1/checkpoint-2500 0 ./results/retrieval_robust false
+
+-> USE /home/work/shared-fi-datasets-01/users/hsiang.chen/Project/Robust/VisRAG/stastic_ret.py stastic results.
+======================
+
+
+## Gen: (should run Ret first and get the embedding and reterival results)
+======================
+1. (Clear)
+python scripts/generate/generate.py \
+    --model_name MiniCPMV2.6 \
+    --model_name_or_path /home/work/shared-fi-datasets-01/users/hsiang.chen/Project/Robust/ModelZoo/openbmb/MiniCPM-V-2_6 \
+    --dataset_name ChartQA \
+    --dataset_name_or_path openbmb/VisRAG-Ret-Test-ChartQA \
+    --rank 0 \
+    --world_size 1 \
+    --topk 3 \
+    --results_root_dir /home/work/shared-fi-datasets-01/users/hsiang.chen/Project/Robust/VisRAG/data/checkpoints/clear \
+    --task_type multi_image \
+    --concatenate_type horizontal \
+    --output_dir /home/work/shared-fi-datasets-01/users/hsiang.chen/Project/Robust/VisRAG/data/checkpoints/generate 
+
+    * model_name: ['MiniCPM', 'MiniCPMV2.0', 'MiniCPMV2.6', 'gpt4o']
+    * model_name_or_path: /home/work/shared-fi-datasets-01/users/hsiang.chen/Project/Robust/ModelZoo/openbmb/MiniCPM-V-2_6
+    * dataset_name: ['ArxivQA', 'ChartQA', 'PlotQA', 'MP-DocVQA', 'SlideVQA', 'InfoVQA']
+    * dataset_name_or_path: 
+    * task_type: ['text', 'page_concatenation', 'weighted_selection', 'multi_image']
+    * concatenate: ['horizontal', 'vertical']
+
+2. (Ret)
+python scripts/generate/generate.py \
+    --model_name MiniCPMV2.6 \
+    --model_name_or_path /home/work/shared-fi-datasets-01/users/hsiang.chen/Project/Robust/ModelZoo/openbmb/MiniCPM-V-2_6 \
+    --dataset_name ChartQA \
+    --dataset_name_or_path openbmb/VisRAG-Ret-Test-ChartQA \
+    --rank 0 \
+    --world_size 1 \
+    --topk 3 \
+    --results_root_dir /home/work/shared-fi-datasets-01/users/hsiang.chen/Project/Robust/VisRAG/data/checkpoints/clear \
+    --task_type multi_image \
+    --concatenate_type horizontal \
+    --output_dir /home/work/shared-fi-datasets-01/users/hsiang.chen/Project/Robust/VisRAG/data/checkpoints/generate 
+
+
+
+# config
+====================================================================
+Evaluating: ArxivQA
+This dataset result dir: /data/checkpoints/eval-2025-09-28-182703-maxq-512-maxp-2048-bsz-16-pooling-wmean-attention-causal-gpus-per-node-1/ArxivQA
+CORPUS_PATH: openbmb/VisRAG-Ret-Test-ArxivQA
+QUERY_PATH: openbmb/VisRAG-Ret-Test-ArxivQA
+QRELS_PATH: openbmb/VisRAG-Ret-Test-ArxivQA
+====================================================================================================
+model_args: ModelArguments(model_name_or_path='/home/work/shared-fi-datasets-01/users/hsiang.chen/Project/Robust/ModelZoo/VisRAG-Ret', target_model_path=None, config_name=None, tokenizer_name=None, processor_name=None, cache_dir='/home/work/shared-fi-datasets-01/users/hsiang.chen/Project/Robust/ModelZoo', untie_encoder=False, feature='last_hidden_state', pooling='wmean', attention='causal', add_linear_head=False, projection_in_dim=768, projection_out_dim=768, dtype='float16', encoder_only=False, pos_token=None, neg_token=None, normalize=True, lora=False, lora_r=32, attn_implementation='sdpa')
+
+data_args: DataArguments(train_dir=None, train_path=None, eval_path=None, qrels_path='openbmb/VisRAG-Ret-Test-MP-DocVQA', query_path='openbmb/VisRAG-Ret-Test-MP-DocVQA', corpus_path='openbmb/VisRAG-Ret-Test-MP-DocVQA', data_dir=None, train_n_passages=8, positive_passage_no_shuffle=False, negative_passage_no_shuffle=False, q_max_len=512, p_max_len=2048, query_instruction=True, corpus_instruction=False, data_cache_dir='/home/work/shared-fi-datasets-01/users/hsiang.chen/Project/Robust/Dataset', query_template='Represent this query for retrieving relevant documents: <query>', query_column_names=None, doc_template='<text>', doc_column_names=None, all_markers=None, encode_as_text_pair=False, from_hf_repo=True)