c6958928eaa0728e8f86d477818d55e904c4c81d569b351f238be73418fef9ff

.gitattributes

@@ -197,3 +197,4 @@ sft/1M3/revise_Full_remoe/logs/0625_1719_llava..._pope_llava_model_args_179bff/m
 sft/1M3/revise_Full_remoe/logs/0625_1719_llava..._pope_llava_model_args_179bff/pope.json filter=lfs diff=lfs merge=lfs -text
 sft/1M3/revise_Full_remoe/logs/0625_1719_llava..._pope_llava_model_args_179bff/submissions/mmbench_en_dev_results.xlsx filter=lfs diff=lfs merge=lfs -text
 sft/1M3/revise_Full_remoe/logs/0625_1719_llava..._pope_llava_model_args_179bff/textvqa_val.json filter=lfs diff=lfs merge=lfs -text
+sft/1M3/revise_Full_remoe/logs/0625_1751_llava...a_val_llava_model_args_179bff/infovqa_val.json filter=lfs diff=lfs merge=lfs -text

sft/1M3/revise_Full_remoe/logs/0625_1751_llava...a_val_llava_model_args_179bff/infovqa_val.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:919df637cc0e9a4231bc4b409d9d6870bfa4cfc858793bca4b4348a654587d48
+size 576443538

sft/1M3/revise_Full_remoe/logs/0625_1751_llava...a_val_llava_model_args_179bff/rank0_metric_eval_done.txt

@@ -0,0 +1 @@
+rank 0 eval done

sft/1M3/revise_Full_remoe/logs/0625_1751_llava...a_val_llava_model_args_179bff/rank1_metric_eval_done.txt

@@ -0,0 +1 @@
+rank 1 eval done

sft/1M3/revise_Full_remoe/logs/0625_1751_llava...a_val_llava_model_args_179bff/rank2_metric_eval_done.txt

@@ -0,0 +1 @@
+rank 2 eval done

sft/1M3/revise_Full_remoe/logs/0625_1751_llava...a_val_llava_model_args_179bff/rank3_metric_eval_done.txt

@@ -0,0 +1 @@
+rank 3 eval done

sft/1M3/revise_Full_remoe/logs/0625_1751_llava...a_val_llava_model_args_179bff/results.json

@@ -0,0 +1,245 @@
+{
+    "results": {
+        "infovqa_val": {
+            "anls,none": 0.26990360585505174,
+            "anls_stderr,none": 0.00838910112249176,
+            "alias": "infovqa_val"
+        },
+        "mmmu_pro": {
+            "mmmu_acc,none": 0.18872999999999998,
+            "mmmu_acc_stderr,none": 0.026738864276958818,
+            "alias": "mmmu_pro"
+        },
+        "mmmu_pro_standard": {
+            "mmmu_acc,none": 0.2422,
+            "mmmu_acc_stderr,none": "N/A",
+            "alias": " - mmmu_pro_standard"
+        },
+        "mmmu_pro_vision": {
+            "mmmu_acc,none": 0.13526,
+            "mmmu_acc_stderr,none": "N/A",
+            "alias": " - mmmu_pro_vision"
+        },
+        "realworldqa": {
+            "exact_match,flexible-extract": 0.4418300653594771,
+            "exact_match_stderr,flexible-extract": 0.01796652860560496,
+            "alias": "realworldqa"
+        }
+    },
+    "groups": {
+        "mmmu_pro": {
+            "mmmu_acc,none": 0.18872999999999998,
+            "mmmu_acc_stderr,none": 0.026738864276958818,
+            "alias": "mmmu_pro"
+        }
+    },
+    "configs": {
+        "infovqa_val": {
+            "task": "infovqa_val",
+            "dataset_path": "lmms-lab/DocVQA",
+            "dataset_name": "InfographicVQA",
+            "dataset_kwargs": {
+                "token": true
+            },
+            "test_split": "validation",
+            "doc_to_visual": "<function infovqa_doc_to_visual at 0x7f3f21688a60>",
+            "doc_to_text": "<function infovqa_doc_to_text at 0x7f3f21688d30>",
+            "doc_to_target": "answers",
+            "description": "",
+            "target_delimiter": " ",
+            "fewshot_delimiter": "\n\n",
+            "metric_list": [
+                {
+                    "metric": "anls",
+                    "aggregation": "mean",
+                    "higher_is_better": true
+                }
+            ],
+            "output_type": "generate_until",
+            "generation_kwargs": {
+                "max_new_tokens": 32,
+                "temperature": 0.0,
+                "do_sample": false,
+                "until": [
+                    "\n\n"
+                ]
+            },
+            "repeats": 1,
+            "should_decontaminate": false,
+            "model_specific_prompt_kwargs": {
+                "default": {
+                    "pre_prompt": "",
+                    "post_prompt": "\nAnswer the question using a single word or phrase."
+                }
+            }
+        },
+        "mmmu_pro_standard": {
+            "task": "mmmu_pro_standard",
+            "dataset_path": "MMMU/MMMU_Pro",
+            "dataset_name": "standard (10 options)",
+            "test_split": "test",
+            "doc_to_visual": "<function mmmu_pro_doc_to_visual at 0x7f3f4cd48820>",
+            "doc_to_text": "<function mmmu_pro_doc_to_text at 0x7f3f4cd525e0>",
+            "doc_to_target": "{{answer}}",
+            "process_results": "<function mmmu_pro_process_results at 0x7f3f4cd5c550>",
+            "description": "",
+            "target_delimiter": " ",
+            "fewshot_delimiter": "\n\n",
+            "metric_list": [
+                {
+                    "metric": "mmmu_acc",
+                    "aggregation": "<function mmmu_pro_aggregate_results at 0x7f3f4cd664c0>",
+                    "higher_is_better": true
+                }
+            ],
+            "output_type": "generate_until",
+            "generation_kwargs": {
+                "max_new_tokens": 256,
+                "until": [
+                    "\n\n"
+                ]
+            },
+            "repeats": 1,
+            "should_decontaminate": false,
+            "metadata": {
+                "version": 0.0,
+                "interleaved_format": false
+            },
+            "model_specific_prompt_kwargs": {
+                "default": {
+                    "pre_prompt": "",
+                    "post_prompt": "Answer with the option letter from the given choices directly."
+                }
+            }
+        },
+        "mmmu_pro_vision": {
+            "task": "mmmu_pro_vision",
+            "dataset_path": "MMMU/MMMU_Pro",
+            "dataset_name": "vision",
+            "test_split": "test",
+            "doc_to_visual": "<function mmmu_pro_doc_to_visual at 0x7f3f4cd36d30>",
+            "doc_to_text": "Answer with the option letter from the given choices directly.",
+            "doc_to_target": "{{answer}}",
+            "process_results": "<function mmmu_pro_process_results at 0x7f3f4cd3dc10>",
+            "description": "",
+            "target_delimiter": " ",
+            "fewshot_delimiter": "\n\n",
+            "metric_list": [
+                {
+                    "metric": "mmmu_acc",
+                    "aggregation": "<function mmmu_pro_aggregate_results at 0x7f3f4cd44b80>",
+                    "higher_is_better": true
+                }
+            ],
+            "output_type": "generate_until",
+            "generation_kwargs": {
+                "max_new_tokens": 256,
+                "until": [
+                    "\n\n"
+                ]
+            },
+            "repeats": 1,
+            "should_decontaminate": false,
+            "metadata": {
+                "version": 0.0,
+                "interleaved_format": false
+            }
+        },
+        "realworldqa": {
+            "task": "realworldqa",
+            "dataset_path": "lmms-lab/RealWorldQA",
+            "dataset_kwargs": {
+                "token": true
+            },
+            "test_split": "test",
+            "doc_to_visual": "<function realworldqa_doc_to_visual at 0x7f3f18164af0>",
+            "doc_to_text": "<function realworldqa_doc_to_text at 0x7f3f1811c160>",
+            "doc_to_target": "answer",
+            "description": "",
+            "target_delimiter": " ",
+            "fewshot_delimiter": "\n\n",
+            "metric_list": [
+                {
+                    "metric": "exact_match",
+                    "aggregation": "mean",
+                    "higher_is_better": true,
+                    "ignore_case": true,
+                    "ignore_punctuation": true
+                }
+            ],
+            "output_type": "generate_until",
+            "generation_kwargs": {
+                "max_new_tokens": 16,
+                "temperature": 0.0,
+                "top_p": 1.0,
+                "num_beams": 1,
+                "do_sample": false,
+                "until": [
+                    "\n\n"
+                ]
+            },
+            "repeats": 1,
+            "filter_list": [
+                {
+                    "name": "flexible-extract",
+                    "filter": [
+                        {
+                            "function": "<class 'utils.NumberWordsToDigitsFilter'>"
+                        },
+                        {
+                            "function": "<class 'utils.MultiChoiceRegexFilter'>",
+                            "group_select": 0,
+                            "ignore_case": true,
+                            "ignore_punctuation": true,
+                            "regex_pattern": "(\\([A-Z]\\))"
+                        }
+                    ]
+                }
+            ],
+            "should_decontaminate": false,
+            "metadata": [
+                {
+                    "version": 0.0
+                }
+            ],
+            "model_specific_prompt_kwargs": {
+                "default": {
+                    "pre_prompt": "",
+                    "post_prompt": ""
+                },
+                "gpt4v": {
+                    "pre_prompt": "",
+                    "post_prompt": ""
+                },
+                "xcomposer2_4khd": {
+                    "pre_prompt": "[UNUSED_TOKEN_146]user\nQuestion: ",
+                    "post_prompt": "[UNUSED_TOKEN_145]\n[UNUSED_TOKEN_146]assistant\nThe answer is"
+                }
+            }
+        }
+    },
+    "versions": {
+        "infovqa_val": "Yaml",
+        "mmmu_pro": "N/A",
+        "mmmu_pro_standard": "Yaml",
+        "mmmu_pro_vision": "Yaml",
+        "realworldqa": "Yaml"
+    },
+    "n-shot": {
+        "infovqa_val": 0,
+        "mmmu_pro": 0,
+        "mmmu_pro_standard": 0,
+        "mmmu_pro_vision": 0,
+        "realworldqa": 0
+    },
+    "model_configs": {
+        "model": "llava",
+        "model_args": "pretrained=/cm/archive/namnv78/checkpoints/Xphi35-siglip224/SMOE/1M3/revise_Full_remoe,conv_template=phi35",
+        "batch_size": "1",
+        "device": null,
+        "limit": null,
+        "bootstrap_iters": 100000,
+        "gen_kwargs": ""
+    },
+    "git_hash": "289c7fe5"
+}

sft/1M3/revise_Full_remoe/logs/0626_0722_llava...image_llava_model_args_179bff/rank0_metric_eval_done.txt

@@ -0,0 +1 @@
+rank 0 eval done

sft/1M3/revise_Full_remoe/logs/0626_0722_llava...image_llava_model_args_179bff/rank1_metric_eval_done.txt

@@ -0,0 +1 @@
+rank 1 eval done

sft/1M3/revise_Full_remoe/logs/0626_0722_llava...image_llava_model_args_179bff/results.json

@@ -0,0 +1,146 @@
+{
+    "results": {
+        "hallusion_bench_image": {
+            "aAcc,none": 42.061,
+            "aAcc_stderr,none": "N/A",
+            "fAcc,none": 16.474,
+            "fAcc_stderr,none": "N/A",
+            "qAcc,none": 12.7473,
+            "qAcc_stderr,none": "N/A",
+            "alias": "hallusion_bench_image"
+        },
+        "mathvista_testmini": {
+            "gpt_eval_score,none": 31.9,
+            "gpt_eval_score_stderr,none": "N/A",
+            "alias": "mathvista_testmini"
+        }
+    },
+    "configs": {
+        "hallusion_bench_image": {
+            "task": "hallusion_bench_image",
+            "dataset_path": "lmms-lab/HallusionBench",
+            "dataset_kwargs": {
+                "token": true
+            },
+            "test_split": "image",
+            "doc_to_visual": "<function hb_doc_to_visual at 0x7f1b6331a1f0>",
+            "doc_to_text": "<function hb_doc_to_text at 0x7f1b6331a940>",
+            "doc_to_target": "gt_answer_details",
+            "process_results": "<function hb_process_results at 0x7f1b63331160>",
+            "description": "",
+            "target_delimiter": " ",
+            "fewshot_delimiter": "\n\n",
+            "metric_list": [
+                {
+                    "metric": "aAcc",
+                    "aggregation": "<function hb_aggregation_result_aAcc at 0x7f1b63331af0>",
+                    "higher_is_better": true
+                },
+                {
+                    "metric": "qAcc",
+                    "aggregation": "<function hb_aggregation_result_qAcc at 0x7f1b6325e160>",
+                    "higher_is_better": true
+                },
+                {
+                    "metric": "fAcc",
+                    "aggregation": "<function hb_aggregation_result_fAcc at 0x7f1b6325e940>",
+                    "higher_is_better": true
+                }
+            ],
+            "output_type": "generate_until",
+            "generation_kwargs": {
+                "max_new_tokens": 128,
+                "temperature": 0.0,
+                "top_p": 1.0,
+                "num_beams": 1,
+                "do_sample": false,
+                "until": [
+                    "\n\n"
+                ]
+            },
+            "repeats": 1,
+            "should_decontaminate": false,
+            "metadata": [
+                {
+                    "version": 0.0
+                }
+            ],
+            "model_specific_prompt_kwargs": {
+                "default": {
+                    "pre_prompt": "",
+                    "post_prompt": ""
+                }
+            }
+        },
+        "mathvista_testmini": {
+            "task": "mathvista_testmini",
+            "dataset_path": "AI4Math/MathVista",
+            "dataset_kwargs": {
+                "token": true
+            },
+            "test_split": "testmini",
+            "doc_to_visual": "<function mathvista_doc_to_visual at 0x7f1b388a5c10>",
+            "doc_to_text": "<function mathvista_doc_to_text at 0x7f1b3849f310>",
+            "doc_to_target": "answer",
+            "process_results": "<function mathvista_process_results at 0x7f1b384a99d0>",
+            "description": "",
+            "target_delimiter": " ",
+            "fewshot_delimiter": "\n\n",
+            "metric_list": [
+                {
+                    "metric": "gpt_eval_score",
+                    "aggregation": "<function mathvista_aggregate_results at 0x7f1b384b50d0>",
+                    "higher_is_better": true
+                }
+            ],
+            "output_type": "generate_until",
+            "generation_kwargs": {
+                "until": [
+                    "ASSISTANT:"
+                ],
+                "max_new_tokens": 1024,
+                "temperature": 0.0,
+                "top_p": 1.0,
+                "num_beams": 1,
+                "do_sample": false,
+                "image_aspect_ratio": "original"
+            },
+            "repeats": 1,
+            "should_decontaminate": false,
+            "model_specific_prompt_kwargs": {
+                "default": {
+                    "shot_type": "format-prompt",
+                    "shot": 0,
+                    "use_caption": false,
+                    "use_ocr": false
+                },
+                "phi3v": {
+                    "shot_type": "solution"
+                }
+            },
+            "model_specific_generation_kwargs": {
+                "llava": {
+                    "image_aspect_ratio": "original"
+                }
+            }
+        }
+    },
+    "versions": {
+        "hallusion_bench_image": "Yaml",
+        "mathvista_testmini": "Yaml"
+    },
+    "n-shot": {
+        "hallusion_bench_image": 0,
+        "mathvista_testmini": 0
+    },
+    "model_configs": {
+        "model": "llava",
+        "model_args": "pretrained=/cm/archive/namnv78/checkpoints/Xphi35-siglip224/SMOE/1M3/revise_Full_remoe,conv_template=phi35",
+        "batch_size": "1",
+        "device": null,
+        "limit": null,
+        "bootstrap_iters": 100000,
+        "gen_kwargs": ""
+    },
+    "git_hash": "289c7fe5"
+}

sft/1M3/revise_Full_remoe/logs/0626_0748_llava...image_llava_model_args_179bff/rank0_metric_eval_done.txt

@@ -0,0 +1 @@
+rank 0 eval done

sft/1M3/revise_Full_remoe/logs/0626_0748_llava...image_llava_model_args_179bff/rank1_metric_eval_done.txt

@@ -0,0 +1 @@
+rank 1 eval done

sft/1M3/revise_Full_remoe/logs/0626_0748_llava...image_llava_model_args_179bff/results.json

@@ -0,0 +1,146 @@
+{
+    "results": {
+        "hallusion_bench_image": {
+            "aAcc,none": 42.3764,
+            "aAcc_stderr,none": "N/A",
+            "fAcc,none": 14.7399,
+            "fAcc_stderr,none": "N/A",
+            "qAcc,none": 12.967,
+            "qAcc_stderr,none": "N/A",
+            "alias": "hallusion_bench_image"
+        },
+        "mathvista_testmini": {
+            "gpt_eval_score,none": 31.9,
+            "gpt_eval_score_stderr,none": "N/A",
+            "alias": "mathvista_testmini"
+        }
+    },
+    "configs": {
+        "hallusion_bench_image": {
+            "task": "hallusion_bench_image",
+            "dataset_path": "lmms-lab/HallusionBench",
+            "dataset_kwargs": {
+                "token": true
+            },
+            "test_split": "image",
+            "doc_to_visual": "<function hb_doc_to_visual at 0x7ff3306fa0d0>",
+            "doc_to_text": "<function hb_doc_to_text at 0x7ff3306fa820>",
+            "doc_to_target": "gt_answer_details",
+            "process_results": "<function hb_process_results at 0x7ff330733040>",
+            "description": "",
+            "target_delimiter": " ",
+            "fewshot_delimiter": "\n\n",
+            "metric_list": [
+                {
+                    "metric": "aAcc",
+                    "aggregation": "<function hb_aggregation_result_aAcc at 0x7ff3307339d0>",
+                    "higher_is_better": true
+                },
+                {
+                    "metric": "qAcc",
+                    "aggregation": "<function hb_aggregation_result_qAcc at 0x7ff3309ac040>",
+                    "higher_is_better": true
+                },
+                {
+                    "metric": "fAcc",
+                    "aggregation": "<function hb_aggregation_result_fAcc at 0x7ff3309ac820>",
+                    "higher_is_better": true
+                }
+            ],
+            "output_type": "generate_until",
+            "generation_kwargs": {
+                "max_new_tokens": 128,
+                "temperature": 0.0,
+                "top_p": 1.0,
+                "num_beams": 1,
+                "do_sample": false,
+                "until": [
+                    "\n\n"
+                ]
+            },
+            "repeats": 1,
+            "should_decontaminate": false,
+            "metadata": [
+                {
+                    "version": 0.0
+                }
+            ],
+            "model_specific_prompt_kwargs": {
+                "default": {
+                    "pre_prompt": "",
+                    "post_prompt": ""
+                }
+            }
+        },
+        "mathvista_testmini": {
+            "task": "mathvista_testmini",
+            "dataset_path": "AI4Math/MathVista",
+            "dataset_kwargs": {
+                "token": true
+            },
+            "test_split": "testmini",
+            "doc_to_visual": "<function mathvista_doc_to_visual at 0x7ff3068a5a60>",
+            "doc_to_text": "<function mathvista_doc_to_text at 0x7ff306149160>",
+            "doc_to_target": "answer",
+            "process_results": "<function mathvista_process_results at 0x7ff306152820>",
+            "description": "",
+            "target_delimiter": " ",
+            "fewshot_delimiter": "\n\n",
+            "metric_list": [
+                {
+                    "metric": "gpt_eval_score",
+                    "aggregation": "<function mathvista_aggregate_results at 0x7ff30615bee0>",
+                    "higher_is_better": true
+                }
+            ],
+            "output_type": "generate_until",
+            "generation_kwargs": {
+                "until": [
+                    "ASSISTANT:"
+                ],
+                "max_new_tokens": 1024,
+                "temperature": 0.0,
+                "top_p": 1.0,
+                "num_beams": 1,
+                "do_sample": false,
+                "image_aspect_ratio": "original"
+            },
+            "repeats": 1,
+            "should_decontaminate": false,
+            "model_specific_prompt_kwargs": {
+                "default": {
+                    "shot_type": "format-prompt",
+                    "shot": 0,
+                    "use_caption": false,
+                    "use_ocr": false
+                },
+                "phi3v": {
+                    "shot_type": "solution"
+                }
+            },
+            "model_specific_generation_kwargs": {
+                "llava": {
+                    "image_aspect_ratio": "original"
+                }
+            }
+        }
+    },
+    "versions": {
+        "hallusion_bench_image": "Yaml",
+        "mathvista_testmini": "Yaml"
+    },
+    "n-shot": {
+        "hallusion_bench_image": 0,
+        "mathvista_testmini": 0
+    },
+    "model_configs": {
+        "model": "llava",
+        "model_args": "pretrained=/cm/archive/namnv78/checkpoints/Xphi35-siglip224/SMOE/1M3/revise_Full_remoe,conv_template=phi35",
+        "batch_size": "1",
+        "device": null,
+        "limit": null,
+        "bootstrap_iters": 100000,
+        "gen_kwargs": ""
+    },
+    "git_hash": "289c7fe5"
+}

sft/1M3/revise_Full_remoe/logs/0626_0910_llava...image_llava_model_args_179bff/rank0_metric_eval_done.txt

@@ -0,0 +1 @@
+rank 0 eval done

sft/1M3/revise_Full_remoe/logs/0626_0910_llava...image_llava_model_args_179bff/rank1_metric_eval_done.txt

@@ -0,0 +1 @@
+rank 1 eval done

sft/1M3/revise_Full_remoe/logs/0626_0910_llava...image_llava_model_args_179bff/results.json

@@ -0,0 +1,146 @@
+{
+    "results": {
+        "hallusion_bench_image": {
+            "aAcc,none": 41.9558,
+            "aAcc_stderr,none": "N/A",
+            "fAcc,none": 15.0289,
+            "fAcc_stderr,none": "N/A",
+            "qAcc,none": 12.967,
+            "qAcc_stderr,none": "N/A",
+            "alias": "hallusion_bench_image"
+        },
+        "mathvista_testmini": {
+            "gpt_eval_score,none": 31.9,
+            "gpt_eval_score_stderr,none": "N/A",
+            "alias": "mathvista_testmini"
+        }
+    },
+    "configs": {
+        "hallusion_bench_image": {
+            "task": "hallusion_bench_image",
+            "dataset_path": "lmms-lab/HallusionBench",
+            "dataset_kwargs": {
+                "token": true
+            },
+            "test_split": "image",
+            "doc_to_visual": "<function hb_doc_to_visual at 0x7f11359811f0>",
+            "doc_to_text": "<function hb_doc_to_text at 0x7f1135981940>",
+            "doc_to_target": "gt_answer_details",
+            "process_results": "<function hb_process_results at 0x7f11358e7160>",
+            "description": "",
+            "target_delimiter": " ",
+            "fewshot_delimiter": "\n\n",
+            "metric_list": [
+                {
+                    "metric": "aAcc",
+                    "aggregation": "<function hb_aggregation_result_aAcc at 0x7f11358e7af0>",
+                    "higher_is_better": true
+                },
+                {
+                    "metric": "qAcc",
+                    "aggregation": "<function hb_aggregation_result_qAcc at 0x7f1135851160>",
+                    "higher_is_better": true
+                },
+                {
+                    "metric": "fAcc",
+                    "aggregation": "<function hb_aggregation_result_fAcc at 0x7f1135851940>",
+                    "higher_is_better": true
+                }
+            ],
+            "output_type": "generate_until",
+            "generation_kwargs": {
+                "max_new_tokens": 128,
+                "temperature": 0.0,
+                "top_p": 1.0,
+                "num_beams": 1,
+                "do_sample": false,
+                "until": [
+                    "\n\n"
+                ]
+            },
+            "repeats": 1,
+            "should_decontaminate": false,
+            "metadata": [
+                {
+                    "version": 0.0
+                }
+            ],
+            "model_specific_prompt_kwargs": {
+                "default": {
+                    "pre_prompt": "",
+                    "post_prompt": ""
+                }
+            }
+        },
+        "mathvista_testmini": {
+            "task": "mathvista_testmini",
+            "dataset_path": "AI4Math/MathVista",
+            "dataset_kwargs": {
+                "token": true
+            },
+            "test_split": "testmini",
+            "doc_to_visual": "<function mathvista_doc_to_visual at 0x7f110aa65c10>",
+            "doc_to_text": "<function mathvista_doc_to_text at 0x7f110a65f310>",
+            "doc_to_target": "answer",
+            "process_results": "<function mathvista_process_results at 0x7f110a6679d0>",
+            "description": "",
+            "target_delimiter": " ",
+            "fewshot_delimiter": "\n\n",
+            "metric_list": [
+                {
+                    "metric": "gpt_eval_score",
+                    "aggregation": "<function mathvista_aggregate_results at 0x7f110a6730d0>",
+                    "higher_is_better": true
+                }
+            ],
+            "output_type": "generate_until",
+            "generation_kwargs": {
+                "until": [
+                    "ASSISTANT:"
+                ],
+                "max_new_tokens": 1024,
+                "temperature": 0.0,
+                "top_p": 1.0,
+                "num_beams": 1,
+                "do_sample": false,
+                "image_aspect_ratio": "original"
+            },
+            "repeats": 1,
+            "should_decontaminate": false,
+            "model_specific_prompt_kwargs": {
+                "default": {
+                    "shot_type": "format-prompt",
+                    "shot": 0,
+                    "use_caption": false,
+                    "use_ocr": false
+                },
+                "phi3v": {
+                    "shot_type": "solution"
+                }
+            },
+            "model_specific_generation_kwargs": {
+                "llava": {
+                    "image_aspect_ratio": "original"
+                }
+            }
+        }
+    },
+    "versions": {
+        "hallusion_bench_image": "Yaml",
+        "mathvista_testmini": "Yaml"
+    },
+    "n-shot": {
+        "hallusion_bench_image": 0,
+        "mathvista_testmini": 0
+    },
+    "model_configs": {
+        "model": "llava",
+        "model_args": "pretrained=/cm/archive/namnv78/checkpoints/Xphi35-siglip224/SMOE/1M3/revise_Full_remoe,conv_template=phi35",
+        "batch_size": "1",
+        "device": null,
+        "limit": null,
+        "bootstrap_iters": 100000,
+        "gen_kwargs": ""
+    },
+    "git_hash": "289c7fe5"
+}

sft/1M3/revise_Full_remoe/logs/0626_1006_llava...image_llava_model_args_179bff/gpt_response/hallusion_output_vd_model.json

@@ -0,0 +1,82 @@
+[
+    {
+        "category": "VD",
+        "subcategory": "illusion",
+        "visual_input": "1",
+        "set_id": "0",
+        "figure_id": "0",
+        "sample_note": "circle",
+        "question_id": "0",
+        "question": "Is the right orange circle the same size as the left orange circle?",
+        "gt_answer_details": "The right orange circle is the same size as the left orange circle.",
+        "gt_answer": "1",
+        "filename": "./VD/illusion/0_0.png",
+        "model_prediction": "no",
+        "gpt4v_output_gpt_check": "0",
+        "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:Is the right orange circle the same size as the left orange circle?\nReference answer: The right orange circle is the same size as the left orange circle.\nPrediction answer:no\nOutput:incorrect"
+    },
+    {
+        "category": "VD",
+        "subcategory": "illusion",
+        "visual_input": "2",
+        "set_id": "0",
+        "figure_id": "1",
+        "sample_note": "circle",
+        "question_id": "1",
+        "question": "Is the right orange circle larger than the left orange circle?",
+        "gt_answer_details": "The right orange circle is larger than the left orange circle.",
+        "gt_answer": "1",
+        "filename": "./VD/illusion/0_1.png",
+        "model_prediction": "Yes, the right orange circle is larger than the left orange circle.",
+        "gpt4v_output_gpt_check": "1",
+        "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:Is the right orange circle larger than the left orange circle?\nReference answer: The right orange circle is larger than the left orange circle.\nPrediction answer:Yes, the right orange circle is larger than the left orange circle.\nOutput:correct"
+    },
+    {
+        "category": "VD",
+        "subcategory": "illusion",
+        "visual_input": "1",
+        "set_id": "1",
+        "figure_id": "0",
+        "sample_note": "box",
+        "question_id": "2",
+        "question": "Is the right grey box the same size as the left grey box?",
+        "gt_answer_details": "The right grey box is the same size as the left grey box.",
+        "gt_answer": "1",
+        "filename": "./VD/illusion/1_0.png",
+        "model_prediction": "Yes, the right grey box is the same size as the left grey box.",
+        "gpt4v_output_gpt_check": "1",
+        "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:Is the right grey box the same size as the left grey box?\nReference answer: The right grey box is the same size as the left grey box.\nPrediction answer:Yes, the right grey box is the same size as the left grey box.\nOutput:correct"
+    },
+    {
+        "category": "VD",
+        "subcategory": "illusion",
+        "visual_input": "1",
+        "set_id": "2",
+        "figure_id": "0",
+        "sample_note": "line",
+        "question_id": "0",
+        "question": "Is the top line longer than the bottom line?",
+        "gt_answer_details": "The top line is the same size as the bottom line.",
+        "gt_answer": "0",
+        "filename": "./VD/illusion/2_0.png",
+        "model_prediction": "Yes, the top line is longer than the bottom line.",
+        "gpt4v_output_gpt_check": "0",
+        "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:Is the top line longer than the bottom line?\nReference answer: The top line is the same size as the bottom line.\nPrediction answer:Yes, the top line is longer than the bottom line.\nOutput:incorrect"
+    },
+    {
+        "category": "VD",
+        "subcategory": "illusion",
+        "visual_input": "2",
+        "set_id": "2",
+        "figure_id": "1",
+        "sample_note": "line",
+        "question_id": "1",
+        "question": "Is the top line shorter than the bottom line?",
+        "gt_answer_details": "The top line is longer than the bottom line.",
+        "gt_answer": "0",
+        "filename": "./VD/illusion/2_1.png",
+        "model_prediction": "Yes, the top line is shorter than the bottom line.",
+        "gpt4v_output_gpt_check": "0",
+        "gpt_answer": "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. If the prediction answer does not conflict with the reference answer, please generate \u201ccorrect\u201d. If the prediction answer conflict with the reference answer, please generate \u201cincorrect\u201d. If the prediction answer is unclear about the answer, please generate \"unclear\". \n\n Question:Is the top line shorter than the bottom line?\nReference answer: The top line is longer than the bottom line.\nPrediction answer:Yes, the top line is shorter than the bottom line.\nOutput:incorrect"
+    }
+]

sft/1M3/revise_Full_remoe/logs/0626_1006_llava...image_llava_model_args_179bff/rank1_metric_eval_done.txt

@@ -0,0 +1 @@
+rank 1 eval done

sft/1M3/revise_Full_remoe/logs/0626_1006_llava...image_llava_model_args_179bff/rank2_metric_eval_done.txt

@@ -0,0 +1 @@
+rank 2 eval done

sft/1M3/revise_Full_remoe/logs/0626_1006_llava...image_llava_model_args_179bff/rank3_metric_eval_done.txt

@@ -0,0 +1 @@
+rank 3 eval done

sft/1M3/revise_Full_remoe/logs/0626_1014_llava...image_llava_model_args_179bff/rank0_metric_eval_done.txt

@@ -0,0 +1 @@
+rank 0 eval done

sft/1M3/revise_Full_remoe/logs/0626_1014_llava...image_llava_model_args_179bff/rank1_metric_eval_done.txt

@@ -0,0 +1 @@
+rank 1 eval done

sft/1M3/revise_Full_remoe/logs/0626_1014_llava...image_llava_model_args_179bff/results.json

@@ -0,0 +1,146 @@
+{
+    "results": {
+        "hallusion_bench_image": {
+            "aAcc,none": 42.3764,
+            "aAcc_stderr,none": "N/A",
+            "fAcc,none": 14.7399,
+            "fAcc_stderr,none": "N/A",
+            "qAcc,none": 12.967,
+            "qAcc_stderr,none": "N/A",
+            "alias": "hallusion_bench_image"
+        },
+        "mathvista_testmini": {
+            "gpt_eval_score,none": 32.2,
+            "gpt_eval_score_stderr,none": "N/A",
+            "alias": "mathvista_testmini"
+        }
+    },
+    "configs": {
+        "hallusion_bench_image": {
+            "task": "hallusion_bench_image",
+            "dataset_path": "lmms-lab/HallusionBench",
+            "dataset_kwargs": {
+                "token": true
+            },
+            "test_split": "image",
+            "doc_to_visual": "<function hb_doc_to_visual at 0x7f5490024160>",
+            "doc_to_text": "<function hb_doc_to_text at 0x7f54900248b0>",
+            "doc_to_target": "gt_answer_details",
+            "process_results": "<function hb_process_results at 0x7f548ff1a0d0>",
+            "description": "",
+            "target_delimiter": " ",
+            "fewshot_delimiter": "\n\n",
+            "metric_list": [
+                {
+                    "metric": "aAcc",
+                    "aggregation": "<function hb_aggregation_result_aAcc at 0x7f548ff1aa60>",
+                    "higher_is_better": true
+                },
+                {
+                    "metric": "qAcc",
+                    "aggregation": "<function hb_aggregation_result_qAcc at 0x7f548fef00d0>",
+                    "higher_is_better": true
+                },
+                {
+                    "metric": "fAcc",
+                    "aggregation": "<function hb_aggregation_result_fAcc at 0x7f548fef08b0>",
+                    "higher_is_better": true
+                }
+            ],
+            "output_type": "generate_until",
+            "generation_kwargs": {
+                "max_new_tokens": 128,
+                "temperature": 0.0,
+                "top_p": 1.0,
+                "num_beams": 1,
+                "do_sample": false,
+                "until": [
+                    "\n\n"
+                ]
+            },
+            "repeats": 1,
+            "should_decontaminate": false,
+            "metadata": [
+                {
+                    "version": 0.0
+                }
+            ],
+            "model_specific_prompt_kwargs": {
+                "default": {
+                    "pre_prompt": "",
+                    "post_prompt": ""
+                }
+            }
+        },
+        "mathvista_testmini": {
+            "task": "mathvista_testmini",
+            "dataset_path": "AI4Math/MathVista",
+            "dataset_kwargs": {
+                "token": true
+            },
+            "test_split": "testmini",
+            "doc_to_visual": "<function mathvista_doc_to_visual at 0x7f54654a7b80>",
+            "doc_to_text": "<function mathvista_doc_to_text at 0x7f54650a0280>",
+            "doc_to_target": "answer",
+            "process_results": "<function mathvista_process_results at 0x7f54650a8940>",
+            "description": "",
+            "target_delimiter": " ",
+            "fewshot_delimiter": "\n\n",
+            "metric_list": [
+                {
+                    "metric": "gpt_eval_score",
+                    "aggregation": "<function mathvista_aggregate_results at 0x7f54650b5040>",
+                    "higher_is_better": true
+                }
+            ],
+            "output_type": "generate_until",
+            "generation_kwargs": {
+                "until": [
+                    "ASSISTANT:"
+                ],
+                "max_new_tokens": 1024,
+                "temperature": 0.0,
+                "top_p": 1.0,
+                "num_beams": 1,
+                "do_sample": false,
+                "image_aspect_ratio": "original"
+            },
+            "repeats": 1,
+            "should_decontaminate": false,
+            "model_specific_prompt_kwargs": {
+                "default": {
+                    "shot_type": "format-prompt",
+                    "shot": 0,
+                    "use_caption": false,
+                    "use_ocr": false
+                },
+                "phi3v": {
+                    "shot_type": "solution"
+                }
+            },
+            "model_specific_generation_kwargs": {
+                "llava": {
+                    "image_aspect_ratio": "original"
+                }
+            }
+        }
+    },
+    "versions": {
+        "hallusion_bench_image": "Yaml",
+        "mathvista_testmini": "Yaml"
+    },
+    "n-shot": {
+        "hallusion_bench_image": 0,
+        "mathvista_testmini": 0
+    },
+    "model_configs": {
+        "model": "llava",
+        "model_args": "pretrained=/cm/archive/namnv78/checkpoints/Xphi35-siglip224/SMOE/1M3/revise_Full_remoe,conv_template=phi35",
+        "batch_size": "1",
+        "device": null,
+        "limit": null,
+        "bootstrap_iters": 100000,
+        "gen_kwargs": ""
+    },
+    "git_hash": "289c7fe5"
+}

sft/1M3/revise_Full_remoe/logs/0626_1036_llava..._plus_llava_model_args_179bff/rank0_metric_eval_done.txt

@@ -0,0 +1 @@
+rank 0 eval done

sft/1M3/revise_Full_remoe/logs/0626_1036_llava..._plus_llava_model_args_179bff/rank1_metric_eval_done.txt

@@ -0,0 +1 @@
+rank 1 eval done

sft/1M3/revise_Full_remoe/logs/0626_1036_llava..._plus_llava_model_args_179bff/results.json

@@ -0,0 +1,168 @@
+{
+    "results": {
+        "chartqa": {
+            "relaxed_overall,none": 0.1252,
+            "relaxed_overall_stderr,none": 0.0066202352681760356,
+            "relaxed_human_split,none": 0.1344,
+            "relaxed_human_split_stderr,none": 0.009651104965839433,
+            "relaxed_augmented_split,none": 0.116,
+            "relaxed_augmented_split_stderr,none": 0.009060953631079097,
+            "alias": "chartqa"
+        },
+        "seedbench_2_plus": {
+            "seedbench_2_plus_Chart,none": 0.4617283950617284,
+            "seedbench_2_plus_Chart_stderr,none": "N/A",
+            "seedbench_2_plus_all,none": 0.4782608695652174,
+            "seedbench_2_plus_all_stderr,none": "N/A",
+            "seedbench_2_plus_Web,none": 0.5287878787878788,
+            "seedbench_2_plus_Web_stderr,none": "N/A",
+            "seedbench_2_plus_Map,none": 0.45353159851301117,
+            "seedbench_2_plus_Map_stderr,none": "N/A",
+            "alias": "seedbench_2_plus"
+        }
+    },
+    "configs": {
+        "chartqa": {
+            "task": "chartqa",
+            "dataset_path": "lmms-lab/ChartQA",
+            "dataset_kwargs": {
+                "token": true
+            },
+            "test_split": "test",
+            "doc_to_visual": "<function chartqa_doc_to_visual at 0x7f1006b5fdc0>",
+            "doc_to_text": "<function chartqa_doc_to_text at 0x7f1006b6a820>",
+            "doc_to_target": "answer",
+            "process_results": "<function chartqa_process_results at 0x7f1006b6aaf0>",
+            "description": "",
+            "target_delimiter": " ",
+            "fewshot_delimiter": "\n\n",
+            "metric_list": [
+                {
+                    "metric": "relaxed_overall",
+                    "aggregation": "mean",
+                    "higher_is_better": true
+                },
+                {
+                    "metric": "relaxed_human_split",
+                    "aggregation": "mean",
+                    "higher_is_better": true
+                },
+                {
+                    "metric": "relaxed_augmented_split",
+                    "aggregation": "mean",
+                    "higher_is_better": true
+                }
+            ],
+            "output_type": "generate_until",
+            "generation_kwargs": {
+                "max_new_tokens": 16,
+                "temperature": 0.0,
+                "do_sample": false,
+                "until": [
+                    "\n\n"
+                ]
+            },
+            "repeats": 1,
+            "should_decontaminate": false,
+            "metadata": [
+                {
+                    "version": 0.0
+                }
+            ],
+            "model_specific_prompt_kwargs": {
+                "default": {
+                    "pre_prompt": "",
+                    "post_prompt": "\nAnswer the question with a single word."
+                },
+                "qwen_vl": {
+                    "pre_prompt": "",
+                    "post_prompt": " Answer:"
+                }
+            }
+        },
+        "seedbench_2_plus": {
+            "task": "seedbench_2_plus",
+            "dataset_path": "doolayer/SEED-Bench-2-Plus",
+            "dataset_kwargs": {
+                "token": true
+            },
+            "test_split": "test",
+            "doc_to_visual": "<function seed_doc_to_visual at 0x7f10209cdd30>",
+            "doc_to_text": "<function seed_doc_to_text at 0x7f10208ec3a0>",
+            "doc_to_target": "answer",
+            "process_results": "<function seed_process_result at 0x7f10208ec8b0>",
+            "description": "",
+            "target_delimiter": " ",
+            "fewshot_delimiter": "\n\n",
+            "metric_list": [
+                {
+                    "metric": "seedbench_2_plus_Chart",
+                    "aggregation": "<function seed_aggregation_result at 0x7f10208ecdc0>",
+                    "higher_is_better": true
+                },
+                {
+                    "metric": "seedbench_2_plus_Map",
+                    "aggregation": "<function seed_aggregation_result at 0x7f102090d280>",
+                    "higher_is_better": true
+                },
+                {
+                    "metric": "seedbench_2_plus_Web",
+                    "aggregation": "<function seed_aggregation_result at 0x7f102090d700>",
+                    "higher_is_better": true
+                },
+                {
+                    "metric": "seedbench_2_plus_all",
+                    "aggregation": "<function seed_aggregation_result at 0x7f102090db80>",
+                    "higher_is_better": true
+                }
+            ],
+            "output_type": "generate_until",
+            "generation_kwargs": {
+                "until": [
+                    "ASSISTANT:"
+                ],
+                "max_new_tokens": 16,
+                "image_aspect_ratio": "original"
+            },
+            "repeats": 1,
+            "should_decontaminate": false,
+            "metadata": [
+                {
+                    "version": 0.0
+                }
+            ],
+            "model_specific_prompt_kwargs": {
+                "llava": {
+                    "img_token": "<image>",
+                    "post_prompt": "Answer with the option's letter from the given choices directly."
+                },
+                "gpt4V": {
+                    "img_token": "<image>",
+                    "post_prompt": "Answer with the option's letter from the given choices directly."
+                },
+                "default": {
+                    "img_token": "<image>",
+                    "post_prompt": "Answer with the option's letter from the given choices directly."
+                }
+            }
+        }
+    },
+    "versions": {
+        "chartqa": "Yaml",
+        "seedbench_2_plus": "Yaml"
+    },
+    "n-shot": {
+        "chartqa": 0,
+        "seedbench_2_plus": 0
+    },
+    "model_configs": {
+        "model": "llava",
+        "model_args": "pretrained=/cm/archive/namnv78/checkpoints/Xphi35-siglip224/SMOE/1M3/revise_Full_remoe,conv_template=phi35",
+        "batch_size": "1",
+        "device": null,
+        "limit": null,
+        "bootstrap_iters": 100000,
+        "gen_kwargs": ""
+    },
+    "git_hash": "289c7fe5"
+}