Add files using upload-large-folder tool

EAGLE/lmms_eval/tasks/ferret/ferret.yaml

@@ -0,0 +1,39 @@
+dataset_path: lmms-lab/Ferret-Bench
+dataset_kwargs:
+  token: True
+task: "ferret"
+test_split: test
+output_type: generate_until
+doc_to_visual: !function utils.ferret_doc_to_visual
+doc_to_text: !function utils.ferret_doc_to_text
+doc_to_target: "gpt_answer"
+generation_kwargs:
+  until:
+    - "ASSISTANT:"
+  image_aspect_ratio: original
+  max_new_tokens: 1024
+  temperature: 0
+  top_p: 0
+  num_beams: 1
+  do_sample: false
+process_results: !function utils.ferret_process_results
+metric_list:
+  - metric: gpt_eval_ferret_all
+    aggregation: !function utils.ferret_all_aggregation
+    higher_is_better: true
+  - metric: gpt_eval_ferret_refer_desc
+    aggregation: !function utils.ferret_refer_desc_aggregation
+    higher_is_better: true
+  - metric: gpt_eval_ferret_refer_reason
+    aggregation: !function utils.ferret_refer_reason_aggregation
+    higher_is_better: true
+  - metric: gpt_eval_ferret_ground_conv
+    aggregation: !function utils.ferret_ground_conv_aggregation
+    higher_is_better: true
+metadata:
+  version: 0.0
+  gpt_eval_model_name: "gpt-4-0314"
+model_specific_prompt_kwargs:
+  default:
+    pre_prompt: ""
+    post_prompt: ""

EAGLE/lmms_eval/tasks/ferret/rule.json

@@ -0,0 +1,5 @@
+{
+    "refer_desc":  {"role": "Assistant", "prompt": "We would like to request your feedback on the performance of two AI assistants in response to the user question displayed above. The user asks the question about specific region of an image. For your reference, the visual content in the image is represented with five descriptive sentences describing the same image. In addition, specific object locations within the image are given, along with detailed coordinates. These coordinates are in the form of bounding boxes, represented as (x1, y1, x2, y2) with floating numbers ranging from 0 to 1. These values correspond to the top left x, top left y, bottom right x, and bottom right y. Also, the relationships between pairs of objects are provided, in the format of object -> relationship -> subject, where the object/subject are indexed by object id from previous object lists as well as the object names. Also, several region description are given, each describing a box region of image, with detailed coordinates. \nPlease rate the spatial correspondence, helpfulness, relevance, accuracy, level of details of their responses. Each assistant receives an overall score on a scale of 1 to 10, where a higher score indicates better overall performance.\nPlease first output a single line containing only two values indicating the scores for Assistant 1 and 2, respectively. The two scores are separated by a space.\nIn the subsequent line, please provide a comprehensive explanation of your evaluation, avoiding any potential bias and ensuring that the order in which the responses were presented does not affect your judgment."},
+    "refer_reason":  {"role": "Assistant", "prompt": "We would like to request your feedback on the performance of two AI assistants in response to the user question displayed above. The user asks the question about specific region of an image. For your reference, the visual content in the image is represented with five descriptive sentences describing the same image. In addition, specific object locations within the image are given, along with detailed coordinates. These coordinates are in the form of bounding boxes, represented as (x1, y1, x2, y2) with floating numbers ranging from 0 to 1. These values correspond to the top left x, top left y, bottom right x, and bottom right y. Also, the relationships between pairs of objects are provided, in the format of object -> relationship -> subject, where the object/subject are indexed by object id from previous object lists as well as the object names. Also, several region description are given, each describing a box region of image, with detailed coordinates. \nPlease rate the spatial correspondence, helpfulness, relevance, accuracy, level of details of their responses. Each assistant receives an overall score on a scale of 1 to 10, where a higher score indicates better overall performance.\nPlease first output a single line containing only two values indicating the scores for Assistant 1 and 2, respectively. The two scores are separated by a space.\nIn the subsequent line, please provide a comprehensive explanation of your evaluation, avoiding any potential bias and ensuring that the order in which the responses were presented does not affect your judgment."},
+    "ground_conv":  {"role": "Assistant", "prompt": "We would like to request your feedback on the performance of two AI assistants in response to the user question displayed above. The user asks the question that requires model to predict the coordinates of relevant object. For your reference, the visual content in the image is represented with five descriptive sentences describing the same image. In addition, specific object locations within the image are given, along with detailed coordinates. These coordinates are in the form of bounding boxes, represented as (x1, y1, x2, y2) with floating numbers ranging from 0 to 1. These values correspond to the top left x, top left y, bottom right x, and bottom right y. Also, the relationships between pairs of objects are provided, in the format of object -> relationship -> subject, where the object/subject are indexed by object id from previous object lists as well as the object names. Also, several region description are given, each describing a box region of image, with detailed coordinates. \nPlease rate the predicted coordinates, helpfulness, relevance, accuracy, level of details of their responses. Specifically, pay your attention to the precision of the coordinates and whether it matches the object. Small deviation (<20% of ground-truth box width or height) of coordinates is allowed and shouldn't be punished. More than that, the degree of deviation should be reflected in scoring too.  Each assistant receives an overall score on a scale of 1 to 10, where a higher score indicates better overall performance.\nPlease first output a single line containing only two values indicating the scores for Assistant 1 and 2, respectively. The two scores are separated by a space.\nIn the subsequent line, please provide a comprehensive explanation of your evaluation, avoiding any potential bias and ensuring that the order in which the responses were presented does not affect your judgment."}
+}

EAGLE/lmms_eval/tasks/ferret/utils.py

@@ -0,0 +1,206 @@
+import json
+import logging
+import os
+import requests
+import numpy as np
+import openai
+from openai import OpenAI
+import time
+import yaml
+from pathlib import Path
+from copy import deepcopy
+
+eval_logger = logging.getLogger("lmms-eval")
+NUM_SECONDS_TO_SLEEP = 0.5
+
+FERRET_W_METRICS = ["gpt_eval_ferret_refer_desc", "gpt_eval_ferret_refer_reason", "gpt_eval_ferret_ground_conv"]
+
+rule_dict = json.load(open(os.path.join(os.path.dirname(os.path.abspath(__file__)), "rule.json"), "r"))
+
+with open(Path(__file__).parent / "ferret.yaml", "r") as f:
+    raw_data = f.readlines()
+    safe_data = []
+    for i, line in enumerate(raw_data):
+        # remove function definition since yaml load cannot handle it
+        if "!function" not in line:
+            safe_data.append(line)
+
+    config = yaml.safe_load("".join(safe_data))
+
+GPT_EVAL_MODEL_NAME = config["metadata"]["gpt_eval_model_name"]
+
+API_TYPE = os.getenv("API_TYPE", "openai")
+
+if API_TYPE == "openai":
+    API_URL = os.getenv("OPENAI_API_URL", "https://api.openai.com/v1/chat/completions")
+    API_KEY = os.getenv("OPENAI_API_KEY", "YOUR_API_KEY")
+    headers = {
+        "Authorization": f"Bearer {API_KEY}",
+        "Content-Type": "application/json",
+    }
+elif API_TYPE == "azure":
+    API_URL = os.getenv("AZURE_ENDPOINT", "https://api.cognitive.microsoft.com/sts/v1.0/issueToken")
+    API_KEY = os.getenv("AZURE_API_KEY", "YOUR_API_KEY")
+    headers = {
+        "api-key": API_KEY,
+        "Content-Type": "application/json",
+    }
+
+
+def get_eval(content: str, max_tokens: int, retries: int = 3):
+    global headers
+
+    messages = [
+        {
+            "role": "system",
+            "content": "You are a helpful and precise assistant for checking the quality of the answer.",
+        },
+        {"role": "user", "content": content},
+    ]
+
+    payload = {
+        "model": GPT_EVAL_MODEL_NAME,
+        "messages": messages,
+        "temperature": 0.2,
+        "max_tokens": max_tokens,
+    }
+
+    for attempt in range(retries):
+        try:
+            response = requests.post(API_URL, headers=headers, json=payload)
+            response.raise_for_status()
+            response_data = response.json()
+
+            content = response_data["choices"][0]["message"]["content"].strip()
+            if content != "":
+                return content, response_data["model"]
+            break  # If successful, break out of the loop
+
+        except Exception as e:
+            eval_logger.info(f"Attempt {attempt + 1} failed with error: {e}")
+            if attempt < retries - 1:  # If we have retries left, sleep and then continue to next attempt
+                time.sleep(NUM_SECONDS_TO_SLEEP)
+            else:  # If this was the last attempt, log and return empty
+                eval_logger.error(f"All {retries} attempts failed. Last error message: {e}")
+                return "", ""
+    return "", ""
+
+
+def parse_score(review):
+    try:
+        score_pair = review.split("\n")[0]
+        score_pair = score_pair.replace(",", " ")
+        sp = score_pair.split(" ")
+        if len(sp) == 2:
+            return [float(sp[0]), float(sp[1])]
+        else:
+            eval_logger.debug(f"Can not split: {review}. Returning [-1, -1]")
+            return [-1, -1]
+    except Exception as e:
+        eval_logger.debug(f"Error: {e}. Returning [-1, -1]")
+        return [-1, -1]
+
+
+def ferret_doc_to_visual(doc):
+    return [doc["image"].convert("RGB")]
+
+
+def ferret_doc_to_text(doc, model_specific_prompt_kwargs=None):
+    if model_specific_prompt_kwargs is None:
+        model_specific_prompt_kwargs = {}
+    pre_prompt = model_specific_prompt_kwargs.get("pre_prompt", "")
+    post_prompt = model_specific_prompt_kwargs.get("post_prompt", "")
+    question = f"{pre_prompt}{doc['question']}{post_prompt}"
+    return question
+
+
+def ferret_process_results(doc, result):
+    """
+    Args:
+        doc: a instance of the eval dataset
+        results: [pred]
+    Returns:
+        a dictionary with key: metric name (in this case coco_bleu), value: metric value
+    """
+    try:
+        question = doc.get("question", "")
+        ans1 = doc.get("gpt_answer", "")
+        ans2 = result[0] if result else ""
+        context = doc.get("context", [])
+        context = "\n".join(context) if isinstance(context, list) else context
+        category = doc.get("category", "")
+        rule = rule_dict.get(category, {})
+        prompt = rule.get("prompt", "")
+        role = rule.get("role", "user")
+        content = f"[Context]\n{context}\n\n" f"[Question]\n{question}\n\n" f"[{role} 1]\n{ans1}\n\n[End of {role} 1]\n\n" f"[{role} 2]\n{ans2}\n\n[End of {role} 2]\n\n" f"[System]\n{prompt}\n\n"
+        review, model_name = get_eval(content, 1024)
+        scores = parse_score(review)
+    except Exception as e:
+        eval_logger.error(f"Error for Question ID: {doc.get('question_id', 'Unknown')}: {e}")
+        review = "Failed to Get a Proper Review."
+        model_name = "Failed Request"
+        scores = [-1, -1]
+
+    metric = f"gpt_eval_ferret_{doc.get('category', 'all')}"
+    category_review_dict = {
+        "question": question,
+        "ans1": ans1,
+        "ans2": ans2,
+        "context": context,
+        "category": category,
+        "review": review,
+        "scores": scores,
+        "eval_model": model_name,
+    }
+
+    non_category_review_dict = deepcopy(category_review_dict)
+    non_category_review_dict["scores"] = [-999, -999]
+
+    data_dict = {}
+    for m in FERRET_W_METRICS:
+        if m == metric:
+            data_dict[m] = category_review_dict
+        else:
+            data_dict[m] = non_category_review_dict
+    data_dict["gpt_eval_ferret_all"] = category_review_dict
+
+    # return {"gpt_eval_ferret_all": review_dict}
+    return data_dict
+
+
+def ferret_refer_desc_aggregation(results):
+    return ferret_aggregation(results, "refer_desc")
+
+
+def ferret_refer_reason_aggregation(results):
+    return ferret_aggregation(results, "refer_reason")
+
+
+def ferret_ground_conv_aggregation(results):
+    return ferret_aggregation(results, "ground_conv")
+
+
+def ferret_all_aggregation(results):
+    return ferret_aggregation(results, "all")
+
+
+def ferret_aggregation(results, category):
+    try:
+        scores = []
+        for result in results:
+            if -999 in result["scores"]:
+                continue
+            scores.append(result["scores"])
+
+        stats = np.asarray(scores).mean(0).tolist()
+        stats = [round(x, 3) for x in stats]
+        # gpt4_score_percentage = stats[0] * 10
+        # model_score_percentage = stats[1] * 10
+        # eval_logger.info(f"Category: {category}")
+        # eval_logger.info(f"GPT4 Score: {gpt4_score_percentage:.1f}%")
+        # eval_logger.info(f"Model Score: {model_score_percentage:.1f}%")
+        # eval_logger.info("=========================")
+        return round(stats[1] / stats[0] * 100, 1)
+    except Exception as e:
+        eval_logger.info(f"Error in ferret_aggregation: {e}, and in category: {category}")
+        return None

EAGLE/lmms_eval/tasks/flickr30k/flickr30k.yaml

@@ -0,0 +1,3 @@
+group: flickr30k
+task:
+- flickr30k_test

EAGLE/lmms_eval/tasks/flickr30k/flickr30k_test.yaml

@@ -0,0 +1,44 @@
+dataset_path: lmms-lab/flickr30k
+dataset_kwargs:
+  token: True
+task : "flickr30k_test"
+test_split: test
+output_type: generate_until
+doc_to_visual: !function utils.flickr_doc_to_visual
+doc_to_text: !function utils.flickr_doc_to_text
+doc_to_target: "answer"
+generation_kwargs:
+  max_new_tokens: 64
+  temperature: 0
+  top_p: 0
+  num_beams: 1
+  do_sample: false
+process_results: !function utils.flickr_process_result
+# Note that the metric name can be either a registed metric function (such as the case for GQA) or a key name returned by process_results
+metric_list:
+  - metric: flickr_Bleu_4 
+    aggregation : !function utils.flickr_bleu4
+    higher_is_better : true
+  - metric: flickr_Bleu_3
+    aggregation : !function utils.flickr_bleu3
+    higher_is_better : true
+  - metric: flickr_Bleu_2
+    aggregation : !function utils.flickr_bleu2
+    higher_is_better : true
+  - metric: flickr_Bleu_1
+    aggregation : !function utils.flickr_bleu1
+    higher_is_better : true
+  - metric: flickr_METEOR
+    aggregation : !function utils.flickr_meteor
+    higher_is_better : true
+  - metric: flickr_ROUGE_L
+    aggregation : !function utils.flickr_rougel
+    higher_is_better : true
+  - metric: flickr_CIDEr
+    aggregation : !function utils.flickr_cider
+    higher_is_better : true
+  #- metric: flickr_SPICE
+  #  aggregation : !function utils.flickr_spice
+  #  higher_is_better : true
+metadata:
+  - version: 0.0

EAGLE/lmms_eval/tasks/flickr30k/utils.py

@@ -0,0 +1,141 @@
+import os
+import json
+from pycocoevalcap.eval import COCOEvalCap, Bleu, Meteor, Rouge, Cider, Spice
+from pycocoevalcap.tokenizer.ptbtokenizer import PTBTokenizer
+from pycocotools.coco import COCO
+from lmms_eval.tasks._task_utils.file_utils import generate_submission_file
+import datetime
+
+import logging
+
+eval_logger = logging.getLogger("lmms-eval")
+
+dir_name = os.path.dirname(os.path.abspath(__file__))
+
+FLICKR_METRICS = ["Bleu_4", "Bleu_3", "Bleu_2", "Bleu_1", "METEOR", "ROUGE_L", "CIDEr"]  # , "SPICE"]
+
+
+def flickr_doc_to_visual(doc):
+    return [doc["image"].convert("RGB")]
+
+
+def flickr_doc_to_text(doc):
+    # question = "Please carefully observe the image and come up with a caption for the image"
+    return f"Provide a one-sentence caption for the provided image."
+
+
+def flickr_process_result(doc, result):
+    """
+    Args:
+        doc: a instance of the eval dataset
+        results: [pred]
+    Returns:
+        a dictionary with key: metric name, value: metric value
+    """
+    pred = result[0] if len(result) > 0 else ""
+    image_id = int(doc["img_id"])
+
+    data_dict = {"answer": doc["caption"], "pred": pred, "image_id": image_id}
+
+    return {f"flickr_{metric}": data_dict for metric in FLICKR_METRICS}
+
+
+def flickr_aggregation_result(results, metric, args):
+    scorers = [(Bleu(4), "Bleu_1"), (Bleu(4), "Bleu_2"), (Bleu(4), "Bleu_3"), (Bleu(4), "Bleu_4"), (Meteor(), "METEOR"), (Rouge(), "ROUGE_L"), (Cider(), "CIDEr"), (Spice(), "SPICE")]
+    scorers_dict = {s[1]: s for s in scorers}
+
+    stored_results = []
+    # In order to make the coco eval tools to successfully create index
+    # We need at least two dict in the dataset
+    # 'annotation' and 'images'
+    # 'annotation' exactly reproduce the original annotation
+    # 'images' however only need the image id which is contained in the file name
+    dataset = {"annotations": [], "images": []}
+    idx = 0
+    for result in results:
+        stored_results.append({"image_id": int(result["image_id"]), "caption": result["pred"]})
+        for a in result["answer"]:
+            dataset["annotations"].append({"image_id": int(result["image_id"]), "caption": a, "id": idx})
+            idx += 1
+        dataset["images"].append({"id": int(result["image_id"])})
+
+    coco = COCO()
+    # Manually create index here
+    coco.dataset = dataset
+    coco.createIndex()
+
+    flickr_result = coco.loadRes(stored_results)
+    flickr_eval = COCOEvalCap(coco, flickr_result)
+
+    imgIds = flickr_eval.params["image_id"]
+    gts = {}
+    res = {}
+    for imgId in imgIds:
+        gts[imgId] = flickr_eval.coco.imgToAnns[imgId]
+        res[imgId] = flickr_eval.cocoRes.imgToAnns[imgId]
+
+    eval_logger.info("tokenization...")
+    tokenizer = PTBTokenizer()
+    gts = tokenizer.tokenize(gts)
+    res = tokenizer.tokenize(res)
+
+    eval_logger.info(f"Computing {metric} scores...")
+
+    score, scores = scorers_dict[metric][0].compute_score(gts, res)
+    # When metric is one of the Bleu, score will be a list
+    if type(score) == list:
+        n = int(metric.split("_")[-1])
+        score = score[n - 1]
+
+    path = generate_submission_file(f"flickr30k_captions_val2014_alg_results_{metric}.json", args)
+
+    eval_logger.info("Storing prediction that can be submitted to the server ...")
+    with open(path, "w") as f:
+        json.dump(stored_results, f, indent=4)
+
+    return score
+
+
+def flickr_bleu4(results, args):
+    return flickr_aggregation_result(results, "Bleu_4", args)
+
+
+def flickr_bleu3(results, args):
+    return flickr_aggregation_result(results, "Bleu_3", args)
+
+
+def flickr_bleu2(results, args):
+    return flickr_aggregation_result(results, "Bleu_2", args)
+
+
+def flickr_bleu1(results, args):
+    return flickr_aggregation_result(results, "Bleu_1", args)
+
+
+def flickr_meteor(results, args):
+    return flickr_aggregation_result(results, "METEOR", args)
+
+
+def flickr_rougel(results, args):
+    return flickr_aggregation_result(results, "ROUGE_L", args)
+
+
+def flickr_cider(results, args):
+    return flickr_aggregation_result(results, "CIDEr", args)
+
+
+def flickr_spice(results, args):
+    return flickr_aggregation_result(results, "SPICE", args)
+
+
+def flickr_test_process_result(doc, result):
+    """
+    Args:
+        doc: a instance of the eval dataset
+        results: [pred]
+    Returns:
+        a dictionary with key: metric name (in this case flickr_passthrough), value: metric value
+    """
+    # The question id in our dataset is the image file itself
+    image_id = doc["img_id"]
+    return {"flickr_passthrough": {"pred": result, "image_id": image_id}}

EAGLE/lmms_eval/tasks/hallusion_bench/evaluate_hb.py

@@ -0,0 +1,129 @@
+import os
+import json
+import logging
+from tqdm import tqdm
+
+from lmms_eval.tasks.hallusion_bench.utils import evaluate_by_chatgpt, check_same_by_chatgpt, assign_correctness, get_eval_all, get_eval_fig, get_eval_pair_all
+
+cur_dir = os.path.dirname(os.path.abspath(__file__))
+output_entry = "model_prediction"
+correctness_entry = "gpt4v_output_gpt_check"
+
+metric = ["aAcc", "fAcc", "qAcc"]
+
+eval_logger = logging.getLogger("lmms-eval")
+
+
+def hb_doc_to_text(doc, model_specific_prompt_kwargs=None):
+    if model_specific_prompt_kwargs is None:
+        model_specific_prompt_kwargs = {}
+    pre_prompt = model_specific_prompt_kwargs.get("pre_prompt", "")
+    post_prompt = model_specific_prompt_kwargs.get("post_prompt", "")
+    return f"{pre_prompt}{doc['question']}{post_prompt}"
+
+
+def hb_doc_to_visual(doc):
+    return [doc["image"].convert("RGB")]
+
+
+def hb_process_results(doc, result):
+    sample = doc
+    # doc.pop("image")
+    sample["model_prediction"] = result[0]
+    return {k: sample for k in metric}
+
+
+def hb_aggregation_result(results, metric, args):
+    data_vd = []
+    data_vs = []
+    for data in tqdm(results, desc="Split vd and vs"):
+        if data["category"] == "VD":
+            data_vd.append(data)
+        if data["category"] == "VS":
+            data_vs.append(data)
+    eval_logger.info("Do gpt eval vd ...")
+    path = os.path.join(args.output_path, "gpt_response")
+    os.makedirs(path, exist_ok=True)
+    save_json_path_vd = f"{path}/hallusion_output_vd_model.json"
+    save_json_path_vs = f"{path}/hallusion_output_vs_model.json"
+    data_vd = evaluate_by_chatgpt(data_vd, output_entry=output_entry, correctness_entry=correctness_entry, load_json=True, save_json_path=save_json_path_vd)
+    # data_vd = check_same_by_chatgpt(data_vd, output_entry=output_entry, load_json=True, save_json_path=save_json_path_vd)
+    data_vd = assign_correctness(data_vd, correctness_entry=correctness_entry)
+    eval_logger.info("Do gpt eval vs")
+    data_vs = evaluate_by_chatgpt(data_vs, output_entry=output_entry, correctness_entry=correctness_entry, load_json=True, save_json_path=save_json_path_vs)
+    # data_vs = check_same_by_chatgpt(data_vs, output_entry=output_entry, load_json=True, save_json_path=save_json_path_vs)
+    data_vs = assign_correctness(data_vs, correctness_entry=correctness_entry)
+    results = data_vs + data_vd
+
+    if metric == "aAcc":
+        all_data = get_eval_all(results, model_correctness_entry=correctness_entry)
+        return round(100 * all_data["correct"] / all_data["total"], 4)
+    elif metric == "fAcc":
+        fig_all = get_eval_fig(results)
+        return round(100 * fig_all["correct"] / fig_all["total"], 4)
+    elif metric == "qAcc":
+        all_data = get_eval_pair_all(results, model_correctness_entry=correctness_entry)
+        return round(100 * all_data["correct"] / all_data["total"], 4)
+
+
+def hb_aggregation_result_qAcc(results, args):
+    return hb_aggregation_result(results, "qAcc", args)
+
+
+def hb_aggregation_result_fAcc(results, args):
+    return hb_aggregation_result(results, "fAcc", args)
+
+
+def hb_aggregation_result_aAcc(results, args):
+    return hb_aggregation_result(results, "aAcc", args)
+
+
+def hb_aggregation_result_intern(results, metric):
+    scores = []
+    for result in results:
+        ans = "1" if result["model_prediction"].lower().find("yes") != -1 else "0"
+        scores.append(ans == result["gt_answer"])
+        result["answer"] = ans
+
+    if metric == "aAcc":
+        return sum(scores) / len(scores)
+    elif metric == "qAcc":
+        qlist = {}
+        for r in results:
+            key = "_".join([r["category"], r["subcategory"], str(r["set_id"]), str(r["question_id"])])
+            try:
+                qlist[key].append(r["answer"] == r["gt_answer"])
+            except:
+                qlist[key] = [r["answer"] == r["gt_answer"]]
+        out = []
+        for q, v in qlist.items():
+            out.append(min(v))
+
+        return sum(out) / len(out)
+    elif metric == "fAcc":
+        qlist = {}
+        for r in results:
+            key = "_".join([r["category"], r["subcategory"], str(r["set_id"]), str(r["figure_id"])])
+            try:
+                qlist[key].append(r["answer"] == r["gt_answer"])
+            except:
+                qlist[key] = [r["answer"] == r["gt_answer"]]
+        out = []
+        for q, v in qlist.items():
+            out.append(min(v))
+        return sum(out) / len(out)
+
+
+def hb_aggregation_result_qAcc_intern(results):
+    eval_logger.info("Calculating qAcc ...")
+    return hb_aggregation_result_intern(results, "qAcc")
+
+
+def hb_aggregation_result_fAcc_intern(results):
+    eval_logger.info("Calculating fAcc ...")
+    return hb_aggregation_result_intern(results, "fAcc")
+
+
+def hb_aggregation_result_aAcc_intern(results):
+    eval_logger.info("Calculating aAcc ...")
+    return hb_aggregation_result_intern(results, "aAcc")

EAGLE/lmms_eval/tasks/hallusion_bench/hallusion_bench_image.yaml

@@ -0,0 +1,41 @@
+dataset_path: lmms-lab/HallusionBench
+dataset_kwargs:
+  token: True
+task: "hallusion_bench_image"
+test_split: image
+output_type: generate_until
+doc_to_visual: !function evaluate_hb.hb_doc_to_visual
+doc_to_text: !function evaluate_hb.hb_doc_to_text
+doc_to_target: "gt_answer_details"
+process_results: !function evaluate_hb.hb_process_results
+model_specific_prompt_kwargs:
+  default:
+    pre_prompt: ""
+    post_prompt: ""
+generation_kwargs:
+  max_new_tokens: 128
+  temperature: 0
+  top_p: 0
+  num_beams: 1
+  do_sample: false
+metric_list:
+  - metric: aAcc
+    aggregation: !function evaluate_hb.hb_aggregation_result_aAcc
+    higher_is_better: true
+  - metric: qAcc
+    aggregation: !function evaluate_hb.hb_aggregation_result_qAcc
+    higher_is_better: true
+  - metric: fAcc
+    aggregation: !function evaluate_hb.hb_aggregation_result_fAcc
+    higher_is_better: true
+  # - metric: aAcc
+  #  aggregation: !function evaluate_hb.hb_aggregation_result_aAcc_intern
+  #  higher_is_better: true
+  # - metric: qAcc
+  #  aggregation: !function evaluate_hb.hb_aggregation_result_qAcc_intern
+  #  higher_is_better: true
+  # - metric: fAcc
+  #  aggregation: !function evaluate_hb.hb_aggregation_result_fAcc_intern
+  #  higher_is_better: true
+metadata:
+  - version: 0.0

EAGLE/lmms_eval/tasks/hallusion_bench/utils.py

@@ -0,0 +1,306 @@
+import csv
+import json
+from tqdm import tqdm
+import numpy as np
+import os
+import time
+import openai
+import threading
+import requests
+import logging
+
+API_TYPE = os.getenv("API_TYPE", "openai")
+
+if API_TYPE == "openai":
+    API_URL = os.getenv("OPENAI_API_URL", "https://api.openai.com/v1/chat/completions")
+    API_KEY = os.getenv("OPENAI_API_KEY", "YOUR_API_KEY")
+    headers = {
+        "Authorization": f"Bearer {API_KEY}",
+        "Content-Type": "application/json",
+    }
+elif API_TYPE == "azure":
+    API_URL = os.getenv("AZURE_ENDPOINT", "https://api.cognitive.microsoft.com/sts/v1.0/issueToken")
+    API_KEY = os.getenv("AZURE_API_KEY", "YOUR_API_KEY")
+    headers = {
+        "api-key": API_KEY,
+        "Content-Type": "application/json",
+    }
+
+eval_logger = logging.getLogger("lmms-eval")
+
+
+def evaluate_by_chatgpt(data, output_entry, correctness_entry, gpt_model="gpt-4", load_json=False, save_json_path="./hallusion_output.json", retries=3):
+    if load_json and os.path.exists(save_json_path):
+        with open(save_json_path, "r") as f:
+            output = json.load(f)
+    else:
+        output = []
+    for sample in tqdm(data[len(output) :], desc="Eval by GPT"):
+        prompt = "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. "
+        prompt += 'If the prediction answer does not conflict with the reference answer, please generate “correct”. If the prediction answer conflict with the reference answer, please generate “incorrect”. If the prediction answer is unclear about the answer, please generate "unclear". \n\n Question:'
+        prompt += sample["question"]
+        prompt += "\nReference answer: "
+        prompt += sample["gt_answer_details"]
+        prompt += "\nPrediction answer:"
+        prompt += sample[output_entry]
+        prompt += "\nOutput:"
+
+        # https://github.com/openai/openai-python/issues/322#issuecomment-1767841683
+        for attempt in range(retries):
+            try:
+                messages = [{"role": "user", "content": prompt}]
+                payload = {
+                    "messages": messages,
+                    "max_tokens": 16,
+                }
+                # set model when using openai api_key. Azure api_key does not need model since the endpoint fixed the model.
+                if API_TYPE == "openai":
+                    payload["model"] = gpt_model
+                response = requests.post(API_URL, headers=headers, json=payload, timeout=30)
+                response.raise_for_status()
+                response = response.json()
+                break
+            except Exception as e:
+                eval_logger.info(f"Attempt {attempt + 1} failed with error: {str(e)}")
+                if attempt < retries - 1:  # If we have retries left, sleep and then continue to next attempt
+                    time.sleep(5)
+                else:  # If this was the last attempt, log and return empty
+                    eval_logger.error(f"All {retries} attempts failed. Last error message: {str(e)}")
+        try:
+            output_text = response["choices"][0]["message"]["content"]
+        except Exception as e:
+            eval_logger.info(f"Get error {str(e)} when extracting response")
+            output_text = "unclear"
+
+        if "incorrect" in output_text.lower():
+            gpt_correctness = "0"
+
+        elif "correct" in output_text.lower():
+            gpt_correctness = "1"
+        else:
+            gpt_correctness = "2"
+
+        sample[correctness_entry] = gpt_correctness
+        sample["gpt_answer"] = prompt + output_text
+
+        output.append(sample)
+
+        with open(save_json_path, "w") as f:
+            json.dump(output, f, indent=4)
+
+    return output
+
+
+def check_same_by_chatgpt(data, output_entry, gpt_model="gpt-4", load_json=False, save_json_path="./hallusion_output.json", retries=3):
+    orig_response = {}
+
+    for r in data:
+        if str(r["figure_id"]) == "0":
+            key = "_".join([r["category"], r["subcategory"], str(r["set_id"]), str(r["question_id"])])
+            orig_response[key] = r[output_entry]
+
+    for sample in tqdm(data, desc="Check same by GPT"):
+        if "same" not in sample.keys():
+            key = "_".join([sample["category"], sample["subcategory"], str(sample["set_id"]), str(sample["question_id"])])
+            response2 = orig_response[key]
+
+            prompt = "Imagine you are an intelligent teacher. Thoroughly read the two responses to two different questions. Assess the consistency of the information provided within those two responses. "
+            prompt += "You do not know the specific questions, but you can asssess the consistency among the two responses by checking for logical conflicts if both responses are correct. "
+            prompt += 'If response1 does not conflict with response2, please generate “same”. Otherwise, generate "different". \n\n response1:'
+            prompt += sample[output_entry]
+            prompt += "\nresponse2: "
+            prompt += response2
+            prompt += "\nOutput:"
+
+            # https://github.com/openai/openai-python/issues/322#issuecomment-1767841683
+            for attempt in range(retries):
+                try:
+                    headers = {
+                        "api-key": API_KEY,
+                        "Content-Type": "application/json",
+                    }
+
+                    messages = [{"role": "user", "content": prompt}]
+
+                    payload = {
+                        "model": gpt_model,
+                        "messages": messages,
+                        "max_tokens": 16,
+                    }
+                    response = requests.post(API_URL, headers=headers, json=payload)
+                    response.raise_for_status()
+                    response = response.json()
+
+                    break
+                except Exception as e:
+                    eval_logger.info(f"Attempt {attempt + 1} failed with error: {str(e)}")
+                    if attempt < retries - 1:  # If we have retries left, sleep and then continue to next attempt
+                        time.sleep(5)
+                    else:  # If this was the last attempt, log and return empty
+                        eval_logger.error(f"All {retries} attempts failed. Last error message: {str(e)}")
+
+            try:
+                output_text = response["choices"][0]["message"]["content"]
+            except Exception as e:
+                eval_logger.info(f"Get error {str(e)} when extracting response")
+                output_text = "different"
+
+            gpt_same = "0"
+
+            if "same" in output_text.lower():
+                gpt_same = "1"
+
+            elif "different" in output_text.lower():
+                gpt_same = "0"
+
+            sample["same"] = gpt_same
+
+            with open(save_json_path, "w") as f:
+                json.dump(data, f, indent=4)
+
+    return data
+
+
+def assign_correctness(data_arr, correctness_entry):
+    for r in data_arr:
+        assert int(r[correctness_entry]) == 0 or int(r[correctness_entry]) == 1 or int(r[correctness_entry]) == 2
+        if r["category"] == "VS" and int(r["figure_id"]) == 0:  # if there is no visual supplement and the model does not know, count it as correct
+            r["correct"] = 1 if int(r[correctness_entry]) == 1 or int(r[correctness_entry]) == 2 else 0
+        else:
+            r["correct"] = 1 if int(r[correctness_entry]) == 1 else 0
+    return data_arr
+
+
+def get_eval_fig(data):  # per figure
+    eval_fig_dict = dict()
+
+    for r in data:
+        if r["category"] == "VS" and str(r["figure_id"]) == "0":  # no figure
+            continue
+        name = "_".join([r["category"], r["subcategory"], str(r["set_id"]), str(r["figure_id"])])
+        if name in eval_fig_dict:
+            c, t = eval_fig_dict[name]
+            eval_fig_dict[name] = (c + r["correct"], t + 1)
+        else:
+            eval_fig_dict[name] = (r["correct"], 1)
+
+    eval_fig_stat = {}
+    eval_fig_stat["note"] = "all accuracy per image (consistency test)"
+    eval_fig_stat["total"] = len(eval_fig_dict.keys())
+    eval_fig_stat["correct"] = 0
+    eval_fig_stat["wrong"] = 0
+    eval_fig_stat["inconsistent"] = 0
+    eval_fig_stat["score"] = 0
+
+    for v in eval_fig_dict.values():
+        if v[0] == v[1]:
+            eval_fig_stat["correct"] += 1
+        elif v[0] == 0:
+            eval_fig_stat["wrong"] += 1
+        else:
+            eval_fig_stat["inconsistent"] += 1
+        eval_fig_stat["score"] += v[0] / v[1]
+
+    eval_fig_stat["score"] = eval_fig_stat["score"] / eval_fig_stat["total"]
+    return eval_fig_stat
+
+
+def get_eval_all(data, model_correctness_entry):  # per question
+    eval_all_dict = dict()
+    eval_all_stat = {}
+    eval_all_stat["LH"] = 0
+    eval_all_stat["VI"] = 0
+    eval_all_stat["Mix"] = 0
+
+    for r in data:
+        name = "_".join([r["category"], r["subcategory"], str(r["set_id"]), str(r["figure_id"]), str(r["question_id"])])
+        assert name not in eval_all_dict
+
+        eval_all_dict[name] = r["correct"]
+
+        if str(r["category"]) == "VD":  # VD
+            if str(r["figure_id"]) == "0":
+                if str(r[model_correctness_entry]) == "0" or str(r[model_correctness_entry]) == "2":
+                    eval_all_stat["VI"] += 1
+            else:
+                if str(r[model_correctness_entry]) == "0":
+                    eval_all_stat["Mix"] += 1
+                elif str(r[model_correctness_entry]) == "2":
+                    eval_all_stat["VI"] += 1
+        else:  # VS
+            if str(r["visual_input"]) == "0":  # no visual
+                if str(r[model_correctness_entry]) == "0":
+                    eval_all_stat["LH"] += 1
+            else:  # original visual or modified visual (isual_input == 1 or 2)
+                if str(r[model_correctness_entry]) == "0":
+                    eval_all_stat["Mix"] += 1
+                elif str(r[model_correctness_entry]) == "2":
+                    eval_all_stat["VI"] += 1
+
+    eval_all_stat["note"] = "all accuracy per question"
+    eval_all_stat["total"] = len(eval_all_dict.keys())
+    eval_all_stat["correct"] = np.count_nonzero(list(eval_all_dict.values()))
+    eval_all_stat["wrong"] = eval_all_stat["total"] - eval_all_stat["correct"]
+
+    return eval_all_stat
+
+
+def get_eval_pair_all(data, model_correctness_entry):  # per question pair
+    orig_correctness = dict()
+    counter = 0
+    lh_counter = 0
+    vi_counter = 0
+    both_counter = 0
+
+    for r in data:
+        if str(r["figure_id"]) == "0":
+            key = "_".join([r["category"], r["subcategory"], str(r["set_id"]), str(r["question_id"])])
+            orig_correctness[key] = r[model_correctness_entry]
+
+    get_eval_pair_dict = dict()
+
+    for r in data:
+        name = "_".join([r["category"], r["subcategory"], str(r["set_id"]), str(r["question_id"])])
+        if name in get_eval_pair_dict:
+            c, t = get_eval_pair_dict[name]
+            get_eval_pair_dict[name] = (c + r["correct"], t + 1)
+        else:
+            get_eval_pair_dict[name] = (r["correct"], 1)
+        counter += 1
+
+    eval_all_pair_stat = {}
+    eval_all_pair_stat["note"] = "all accuracy per question pair"
+    eval_all_pair_stat["total"] = len(get_eval_pair_dict.keys())
+    eval_all_pair_stat["total_q"] = counter
+    eval_all_pair_stat["correct"] = 0
+    eval_all_pair_stat["wrong"] = 0
+    eval_all_pair_stat["LH"] = 0
+    eval_all_pair_stat["VI"] = 0
+    eval_all_pair_stat["Mix"] = 0
+
+    eval_all_pair_stat["LH_cg"] = lh_counter
+    eval_all_pair_stat["VI_cg"] = vi_counter
+    eval_all_pair_stat["Mix_cg"] = both_counter
+
+    # for v in get_eval_pair_dict.values():
+    #     if v[0] == v[1]:
+    #         eval_all_pair_stat["correct"] += 1
+    #     else:
+    #         eval_all_pair_stat["wrong"] += 1
+
+    # for v in get_analysis_pair_dict.values():
+    #     if v[0] > 0 and v[1] > 0:
+    #         eval_all_pair_stat["Mix"] += 1
+    #     elif v[0] > 0:
+    #         eval_all_pair_stat["LH"] += 1
+    #     elif v[1] > 0:
+    #         eval_all_pair_stat["VI"] += 1
+
+    for k in get_eval_pair_dict.keys():
+        v = get_eval_pair_dict[k]
+        if v[0] == v[1]:
+            eval_all_pair_stat["correct"] += 1
+        else:
+            eval_all_pair_stat["wrong"] += 1
+
+    return eval_all_pair_stat

EAGLE/lmms_eval/tasks/iconqa/utils.py

@@ -0,0 +1,57 @@
+import json
+import os
+
+
+def options_to_str(options_prompt):
+    option_prompt_str = ""
+    for i, option in enumerate(options_prompt):
+        option_choice = chr(ord("A") + i)
+        option_prompt_str += f"{option_choice}. {option}\n"
+
+    option_prompt_str = option_prompt_str.rstrip("\n")
+    return option_prompt_str
+
+
+def doc_to_visual(doc):
+    image_list = []
+    if "query_image" in doc:
+        image_list.append(doc["query_image"].convert("RGB"))
+    for i in range(5):
+        id = f"choice_image_{i}"
+        if id in doc and doc[id] is not None:
+            image_list.append(doc[id].convert("RGB"))
+    assert len(image_list) < 6, "Maximum 5 images allowed for ICON-QA"
+    return image_list
+
+
+def doc_to_text(doc, model_specific_prompt_kwargs):
+    question = doc["question"]
+    ques_type = doc["ques_type"]
+    options_prompt = []
+
+    if ques_type == "choose_img":
+        options_prompt.append("The first image.")
+        options_prompt.append("The second image.")
+
+        options_str = options_to_str(options_prompt)
+        full_prompt = f"{model_specific_prompt_kwargs['pre_prompt']}{model_specific_prompt_kwargs['statement']}{model_specific_prompt_kwargs['options_statement'].format(question=question, options=options_str)}"
+
+    elif ques_type == "choose_txt":
+        choices = doc["choices"].split(",")
+        for i, choice in enumerate(choices):
+            options_prompt.append(f"{choice}")
+
+        options_str = options_to_str(options_prompt)
+        full_prompt = f"{model_specific_prompt_kwargs['pre_prompt']}{model_specific_prompt_kwargs['statement']}{model_specific_prompt_kwargs['options_statement'].format(question=question, options=options_str)}"
+
+    elif ques_type == "fill_in_blank":
+        full_prompt = f"{model_specific_prompt_kwargs['pre_prompt']}{model_specific_prompt_kwargs['statement']}{model_specific_prompt_kwargs['freeform_statement'].format(question=question)}"
+
+    return full_prompt
+
+
+def test_process_results(doc, results):
+    pred = results[0]
+    questionId = doc["question_id"]
+    answer = doc["answer"]
+    return {"anls": {"questionId": int(questionId), "answer": answer, "pred_answer": pred}}

EAGLE/lmms_eval/tasks/mme/mme.yaml

@@ -0,0 +1,37 @@
+dataset_path: lmms-lab/MME
+dataset_kwargs:
+  token: True
+task: "mme"
+test_split: test
+output_type: generate_until
+doc_to_visual: !function utils.mme_doc_to_visual
+doc_to_text: !function utils.mme_doc_to_text
+doc_to_target: "answer"
+generation_kwargs:
+  max_new_tokens: 16
+  temperature: 0
+  top_p: 0
+  num_beams: 1
+  do_sample: false
+# The return value of process_results will be used by metrics
+process_results: !function utils.mme_process_results
+# Note that the metric name can be either a registed metric function (such as the case for GQA) or a key name returned by process_results
+metric_list:
+  - metric: mme_percetion_score
+    aggregation: !function utils.mme_aggregate_results
+    higher_is_better: true
+  - metric: mme_cognition_score
+    aggregation: !function utils.mme_aggregate_results
+    higher_is_better: true
+model_specific_prompt_kwargs:
+  default:
+    pre_prompt: ""
+    post_prompt: "\nAnswer the question using a single word or phrase."
+  qwen_vl:  
+    pre_prompt: ""
+    post_prompt: " Answer:"
+  otterhd:
+    pre_prompt: ""
+    post_prompt: " Answer:"
+metadata:
+  - version: 0.0

EAGLE/lmms_eval/tasks/mme/utils.py

@@ -0,0 +1,120 @@
+from collections import defaultdict
+import os
+import datetime
+import json
+from lmms_eval.tasks._task_utils.file_utils import generate_submission_file
+
+import logging
+
+eval_logger = logging.getLogger("lmms-eval")
+
+dir_name = os.path.dirname(os.path.abspath(__file__))
+
+eval_type_dict = {
+    "Perception": [
+        "existence",
+        "count",
+        "position",
+        "color",
+        "posters",
+        "celebrity",
+        "scene",
+        "landmark",
+        "artwork",
+        "OCR",
+    ],
+    "Cognition": [
+        "commonsense_reasoning",
+        "numerical_calculation",
+        "text_translation",
+        "code_reasoning",
+    ],
+}
+
+
+replace_prompt = " Please answer yes or no."
+
+
+def mme_doc_to_visual(doc):
+    return [doc["image"].convert("RGB")]
+
+
+def mme_doc_to_text(doc, model_specific_prompt_kwargs=None):
+    question = doc["question"].strip()
+    if "pre_prompt" in model_specific_prompt_kwargs and model_specific_prompt_kwargs["pre_prompt"] != "":
+        question = question.replace(replace_prompt, "")
+        question = f"{model_specific_prompt_kwargs['pre_prompt']}{question}"
+    if "post_prompt" in model_specific_prompt_kwargs and model_specific_prompt_kwargs["post_prompt"] != "":
+        question = question.replace(replace_prompt, "")
+        question = f"{question}{model_specific_prompt_kwargs['post_prompt']}"
+    return question
+
+
+def parse_pred_ans(pred_ans):
+    """Brought from Otter Eval"""
+    pred_ans = pred_ans.lower().strip().replace(".", "")
+    pred_label = None
+    if pred_ans in ["yes", "no"]:
+        pred_label = pred_ans
+    else:
+        prefix_pred_ans = pred_ans[:4]
+        if "yes" in prefix_pred_ans:
+            pred_label = "yes"
+        elif "no" in prefix_pred_ans:
+            pred_label = "no"
+        else:
+            pred_label = "other"
+    return pred_label
+
+
+def mme_process_results(doc, results):
+    """
+    Args:
+        doc: a instance of the eval dataset
+        results: [pred]
+    Returns:
+        a dictionary with key: metric name (in this case mme score), value: metric value
+    """
+    pred = results[0]
+    pred_ans = parse_pred_ans(pred)
+    gt_ans = doc["answer"].lower().strip().replace(".", "")
+    assert gt_ans in ["yes", "no"]
+    assert pred_ans in ["yes", "no", "other"]
+    score = 1.0 if pred_ans == gt_ans else 0.0
+    category = doc["category"]
+    key_name = "mme_percetion_score" if category in eval_type_dict["Perception"] else "mme_cognition_score"
+    # Note: the key name here is very important. It decides which aggregation function will receive the results
+    # We note down the question id/category to help us aggregate the results later
+    return {key_name: {"question_id": doc["question_id"], "category": category, "score": score}}
+
+
+def mme_aggregate_results(results):
+    """
+    Args:
+        results: a list of values returned by process_results
+    Returns:
+        A score
+    """
+    category2score = defaultdict(dict)
+    for result in results:
+        question_id = result["question_id"]
+        score = result["score"]
+        category = result["category"]
+        if question_id not in category2score[category]:
+            category2score[category][question_id] = []
+        category2score[category][question_id].append(score)
+    category2avg_score = {}
+    for category, question2scores in category2score.items():
+        total_score = 0
+        for question_id, scores in question2scores.items():
+            assert len(scores) == 2
+            acc = sum(scores) / len(scores) * 100.0
+            acc_plus = (sum(scores) == 2) * 100.0
+            score = acc_plus + acc
+            total_score += score
+        avg_score = total_score / len(question2scores)
+        category2avg_score[category] = avg_score
+    for category, avg_score in category2avg_score.items():
+        eval_logger.info(f"{category}: {avg_score:.2f}")
+    total_score = sum(category2avg_score.values())
+    return total_score

EAGLE/lmms_eval/tasks/multidocvqa/multidocvqa.yaml

@@ -0,0 +1,4 @@
+group: multidocvqa
+task:
+- multidocvqa_val
+- multidocvqa_test

EAGLE/lmms_eval/tasks/multidocvqa/multidocvqa_test.yaml

@@ -0,0 +1,20 @@
+dataset_path: lmms-lab/MP-DocVQA
+task: "multidocvqa_test"
+test_split: test
+output_type: generate_until
+doc_to_visual: !function utils.multidocvqa_doc_to_visual
+doc_to_text: !function utils.multidocvqa_doc_to_text
+doc_to_target: "answers"
+generation_kwargs:
+  max_new_tokens: 32
+  temperature: 0
+  do_sample: False
+process_results: !function utils.multidocvqa_process_test_results_for_submission
+metric_list:
+  - metric: submission
+    aggregation: !function utils.multidocvqa_test_aggregate_results_for_submission
+model_specific_prompt_kwargs:
+  default:
+    pre_prompt: ""
+    post_prompt: "\nAnswer the question using a single word or phrase."
+  

EAGLE/lmms_eval/tasks/multidocvqa/multidocvqa_val.yaml

@@ -0,0 +1,23 @@
+dataset_path: lmms-lab/MP-DocVQA
+task: "multidocvqa_val"
+test_split: val
+output_type: generate_until
+doc_to_visual: !function utils.multidocvqa_doc_to_visual
+doc_to_text: !function utils.multidocvqa_doc_to_text
+doc_to_target: "answers"
+generation_kwargs:
+  max_new_tokens: 32
+  temperature: 0
+  do_sample: False
+process_results: !function utils.multidocvqa_process_results
+metric_list:
+  - metric: anls
+    aggregation: !function utils.multidocvqa_aggregate_results_anls
+    higher_is_better: true
+  - metric: accuracy
+    aggregation: !function utils.multidocvqa_aggregate_results_accuracy
+    higher_is_better: true
+model_specific_prompt_kwargs:
+  default:
+    pre_prompt: ""
+    post_prompt: "\nAnswer the question using a single word or phrase."

EAGLE/lmms_eval/tasks/multidocvqa/utils.py

@@ -0,0 +1,116 @@
+import os
+import re
+import ast
+import json
+import logging
+from lmms_eval.api.metrics import levenshtein_distance
+from lmms_eval.tasks._task_utils.file_utils import generate_submission_file
+
+lmms_logger = logging.getLogger("lmms-eval")
+
+
+def multidocvqa_doc_to_text(doc, model_specific_prompt_kwargs):
+    question = doc["question"]
+    pre_prompt = model_specific_prompt_kwargs["pre_prompt"]
+    post_prompt = model_specific_prompt_kwargs["post_prompt"]
+
+    return f"{pre_prompt}{question}{post_prompt}"
+
+
+def multidocvqa_doc_to_visual(doc):
+    return [doc[f"image_{i}"].convert("RGB") for i in range(1, 21) if doc[f"image_{i}"] is not None]
+
+
+def multidocvqa_process_results(doc, results):
+    pred_answer = results[0]
+    answer = ast.literal_eval(doc["answers"])
+
+    return {"anls": {"questionId": int(doc["questionId"]), "answer": answer, "pred_answer": pred_answer}, "accuracy": {"questionId": int(doc["questionId"]), "answer": answer, "pred_answer": pred_answer}}
+
+
+def multidocvqa_aggregate_results_anls(results):
+    keys = {k for result in results for k in result}
+    results = {key: [result.get(key, None) for result in results] for key in keys}
+    evaluator = Evaluator(case_sensitive=False)
+    metric = evaluator.get_metrics(results["answer"], results["pred_answer"])
+
+    return sum(metric["anls"]) / len(metric["anls"])
+
+
+def multidocvqa_aggregate_results_accuracy(results):
+    keys = {k for result in results for k in result}
+    results = {key: [result.get(key, None) for result in results] for key in keys}
+    evaluator = Evaluator(case_sensitive=False)
+    metric = evaluator.get_metrics(results["answer"], results["pred_answer"])
+
+    return sum(metric["accuracy"]) / len(metric["accuracy"])
+
+
+def multidocvqa_process_test_results_for_submission(doc, results):
+    answer = results[0]
+    return {"submission": {"questionId": int(doc["questionId"]), "answer": answer, "answer_page": None}}
+
+
+def multidocvqa_test_aggregate_results_for_submission(results, args):
+    path = generate_submission_file("multidocvqa_test_for_submission.json", args)
+    with open(path, "w") as f:
+        json.dump(results, f)
+    lmms_logger.info(f"Results saved to {path}.")
+
+
+##################
+# Helper functions
+##################
+
+
+class Evaluator:
+    def __init__(self, case_sensitive=False):
+        self.case_sensitive = case_sensitive
+        self.get_edit_distance = levenshtein_distance
+        self.anls_threshold = 0.5
+
+    def get_metrics(self, gt_answers, preds):
+        batch_accuracy = []
+        batch_anls = []
+        for batch_idx in range(len(preds)):
+            gt = [self._preprocess_str(gt_elm) for gt_elm in gt_answers[batch_idx]]
+            pred = self._preprocess_str(preds[batch_idx])
+
+            batch_accuracy.append(self._calculate_accuracy(gt, pred))
+            batch_anls.append(self._calculate_anls(gt, pred))
+
+        return {"accuracy": batch_accuracy, "anls": batch_anls}
+
+    def _preprocess_str(self, string):
+        if not self.case_sensitive:
+            string = string.lower()
+
+        return string.strip()
+
+    def _calculate_accuracy(self, gt, pred):
+        if pred == "none":
+            return 0
+
+        for gt_elm in gt:
+            if gt_elm == pred:
+                return 1
+
+        return 0
+
+    def _calculate_anls(self, gt, pred):
+        if len(pred) == 0:
+            return 0
+
+        if pred == "none":
+            return 0
+
+        answers_similarity = [1 - self.get_edit_distance(gt_elm, pred) / max(len(gt_elm), len(pred)) for gt_elm in gt]
+        max_similarity = max(answers_similarity)
+
+        anls = max_similarity if max_similarity >= self.anls_threshold else 0
+        return anls
+
+
+if __name__ == "__main__":
+    print("-----------------")
+    multidocvqa_aggregate_results_anls([{"questionId": 1, "answer": ["answer"], "pred_answer": "pred_answer"}, {"questionId": 2, "answer": ["nswer"], "pred_answer": "nswer"}])

EAGLE/lmms_eval/tasks/nocaps/_default_template_nocaps_yaml

@@ -0,0 +1,3 @@
+model_specific_prompt_kwargs:
+  default:
+    prompt: "Provide a one-sentence caption for the provided image."

EAGLE/lmms_eval/tasks/nocaps/nocaps.yaml

@@ -0,0 +1,4 @@
+group : nocaps
+task:
+  - nocaps_test
+  - nocaps_val

EAGLE/lmms_eval/tasks/nocaps/nocaps_test.yaml

@@ -0,0 +1,25 @@
+dataset_path: lmms-lab/NoCaps
+dataset_kwargs:
+  token: True
+task : "nocaps_test"
+group : "nocaps_caption"
+test_split: test
+output_type: generate_until
+doc_to_visual: !function utils.nocaps_doc_to_visual
+doc_to_text: !function utils.nocaps_doc_to_text
+doc_to_target: "annotations_captions"
+generation_kwargs:
+  max_new_tokens: 64
+  temperature: 0
+  top_p: 0
+  num_beams: 1
+  do_sample: false
+process_results: !function utils.nocaps_test_process_result
+# Note that the metric name can be either a registed metric function (such as the case for GQA) or a key name returned by process_results
+metric_list:
+  - metric: nocaps_passthrough 
+    aggregation : !function utils.nocaps_test_aggregation_result
+    higher_is_better : true
+metadata:
+  - version: 0.0
+include: _default_template_nocaps_yaml

EAGLE/lmms_eval/tasks/nocaps/nocaps_val.yaml

@@ -0,0 +1,46 @@
+dataset_path: lmms-lab/NoCaps
+dataset_kwargs:
+  token: True
+task: "nocaps_val"
+group : "nocaps_caption"
+test_split: validation
+output_type: generate_until
+doc_to_visual: !function utils.nocaps_doc_to_visual
+doc_to_text: !function utils.nocaps_doc_to_text
+doc_to_target: "annotations_captions"
+generation_kwargs:
+  max_new_tokens: 64
+  temperature: 0
+  top_p: 0
+  num_beams: 1
+  do_sample: false
+process_results: !function utils.nocaps_process_result
+# Note that the metric name can be either a registed metric function (such as the case for GQA) or a key name returned by process_results
+metric_list:
+  - metric: nocaps_Bleu_4 
+    aggregation : !function utils.nocaps_bleu4
+    higher_is_better : true
+  - metric: nocaps_Bleu_3
+    aggregation : !function utils.nocaps_bleu3
+    higher_is_better : true
+  - metric: nocaps_Bleu_2
+    aggregation : !function utils.nocaps_bleu2
+    higher_is_better : true
+  - metric: nocaps_Bleu_1
+    aggregation : !function utils.nocaps_bleu1
+    higher_is_better : true
+  - metric: nocaps_METEOR
+    aggregation : !function utils.nocaps_meteor
+    higher_is_better : true
+  - metric: nocaps_ROUGE_L
+    aggregation : !function utils.nocaps_rougel
+    higher_is_better : true
+  - metric: nocaps_CIDEr
+    aggregation : !function utils.nocaps_cider
+    higher_is_better : true
+  #- metric: nocaps_SPICE
+  #  aggregation : !function utils.nocaps_spice
+  #  higher_is_better : true
+metadata:
+  - version: 0.0
+include: _default_template_nocaps_yaml

EAGLE/lmms_eval/tasks/nocaps/utils.py

@@ -0,0 +1,153 @@
+import os
+import json
+from pycocoevalcap.eval import COCOEvalCap, Bleu, Meteor, Rouge, Cider, Spice
+from pycocoevalcap.tokenizer.ptbtokenizer import PTBTokenizer
+from pycocotools.coco import COCO
+
+from lmms_eval.tasks._task_utils.file_utils import generate_submission_file
+
+import logging
+
+eval_logger = logging.getLogger("lmms-eval")
+
+dir_name = os.path.dirname(os.path.abspath(__file__))
+
+NOCAPS_METRICS = ["Bleu_4", "Bleu_3", "Bleu_2", "Bleu_1", "METEOR", "ROUGE_L", "CIDEr"]  # , "SPICE"]
+
+
+def nocaps_doc_to_visual(doc):
+    return [doc["image"].convert("RGB")]
+
+
+def nocaps_doc_to_text(doc, model_specific_prompt_kwargs=None):
+    # question = "Please carefully observe the image and come up with a caption for the image"
+    return model_specific_prompt_kwargs["prompt"]
+
+
+def nocaps_process_result(doc, result):
+    """
+    Args:
+        doc: a instance of the eval dataset
+        results: [pred]
+    Returns:
+        a dictionary with key: metric name, value: metric value
+    """
+    pred = result[0]
+    # The question id in our dataset is the image file itself
+    image_id = doc["image_id"]
+
+    data_dict = {"answer": doc["annotations_captions"], "pred": pred, "image_id": image_id}
+
+    return {f"nocaps_{metric}": data_dict for metric in NOCAPS_METRICS}
+
+
+def nocaps_aggregation_result(results, metric, args=None):
+    scorers = [(Bleu(4), "Bleu_1"), (Bleu(4), "Bleu_2"), (Bleu(4), "Bleu_3"), (Bleu(4), "Bleu_4"), (Meteor(), "METEOR"), (Rouge(), "ROUGE_L"), (Cider(), "CIDEr"), (Spice(), "SPICE")]
+    scorers_dict = {s[1]: s for s in scorers}
+
+    stored_results = []
+    # In order to make the coco eval tools to successfully create index
+    # We need at least two dict in the dataset
+    # 'annotation' and 'images'
+    # 'annotation' exactly reproduce the original annotation
+    # 'images' however only need the image id which is contained in the file name
+    dataset = {"annotations": [], "images": []}
+    idx = 0
+    for result in results:
+        stored_results.append({"image_id": int(result["image_id"]), "caption": result["pred"]})
+        for a in result["answer"]:
+            dataset["annotations"].append({"image_id": int(result["image_id"]), "caption": a, "id": idx})
+            idx += 1
+        dataset["images"].append({"id": result["image_id"]})
+
+    coco = COCO()
+    # Manually create index here
+    coco.dataset = dataset
+    coco.createIndex()
+
+    nocaps_result = coco.loadRes(stored_results)
+    nocaps_eval = COCOEvalCap(coco, nocaps_result)
+
+    imgIds = nocaps_eval.params["image_id"]
+    gts = {}
+    res = {}
+    for imgId in imgIds:
+        gts[imgId] = nocaps_eval.coco.imgToAnns[imgId]
+        res[imgId] = nocaps_eval.cocoRes.imgToAnns[imgId]
+
+    eval_logger.info("tokenization...")
+    tokenizer = PTBTokenizer()
+    gts = tokenizer.tokenize(gts)
+    res = tokenizer.tokenize(res)
+
+    eval_logger.info(f"Computing {metric} scores...")
+
+    score, scores = scorers_dict[metric][0].compute_score(gts, res)
+    # When metric is one of the Bleu, score will be a list
+    if type(score) == list:
+        n = int(metric.split("_")[-1])
+        score = score[n - 1]
+
+    path = generate_submission_file(f"nocaps_val_{metric}_scores.json", args)
+    eval_logger.info("Storing prediction that can be submitted to the server ...")
+    with open(path, "w") as f:
+        json.dump(stored_results, f, indent=4)
+    eval_logger.info(f"Your result has been saved to {path}.")
+
+    return score
+
+
+def nocaps_bleu4(results, args=None):
+    return nocaps_aggregation_result(results, "Bleu_4", args)
+
+
+def nocaps_bleu3(results, args=None):
+    return nocaps_aggregation_result(results, "Bleu_3", args)
+
+
+def nocaps_bleu2(results, args=None):
+    return nocaps_aggregation_result(results, "Bleu_2", args)
+
+
+def nocaps_bleu1(results, args=None):
+    return nocaps_aggregation_result(results, "Bleu_1", args)
+
+
+def nocaps_meteor(results, args=None):
+    return nocaps_aggregation_result(results, "METEOR", args)
+
+
+def nocaps_rougel(results, args=None):
+    return nocaps_aggregation_result(results, "ROUGE_L", args)
+
+
+def nocaps_cider(results, args=None):
+    return nocaps_aggregation_result(results, "CIDEr", args)
+
+
+def nocaps_spice(results, args=None):
+    return nocaps_aggregation_result(results, "SPICE", args)
+
+
+def nocaps_test_process_result(doc, result):
+    """
+    Args:
+        doc: a instance of the eval dataset
+        results: [pred]
+    Returns:
+        a dictionary with key: metric name (in this case nocaps_passthrough), value: metric value
+    """
+    return {"nocaps_passthrough": {"pred": result[0], "image_id": doc["image_id"]}}
+
+
+def nocaps_test_aggregation_result(results, args=None):
+    stored_results = []
+    for result in results:
+        stored_results.append({"image_id": int(result["image_id"]), "caption": result["pred"]})
+
+    path = generate_submission_file("nocaps_captions_nocaps_test_alg_results.json", args)
+    eval_logger.info("Storing prediction that can be submitted to the server ...")
+    with open(path, "w") as f:
+        json.dump(stored_results, f, indent=4)
+
+    eval_logger.info(f"Your test result has been stored in {path}. Make sure you also have the val result stored to submit to the server on https://codalab.lisn.upsaclay.fr/competitions/7404#participate.")

EAGLE/lmms_eval/tasks/ok_vqa/_default_template_vqa_yaml

@@ -0,0 +1,24 @@
+dataset_path: lmms-lab/OK-VQA
+output_type: generate_until
+doc_to_visual: !function utils.ok_vqa_doc_to_visual
+doc_to_text: !function utils.ok_vqa_doc_to_text
+doc_to_target: "answer"
+generation_kwargs:
+  until:
+    - "ASSISTANT:"
+metric_list:
+  - metric: exact_match
+    aggregation: mean
+    higher_is_better: true
+    ignore_case: true
+    ignore_punctuation: true
+  - metric: submission
+    aggregation: !function utils.ok_vqa_aggreate_submissions
+    higher_is_better: true
+process_results: !function utils.ok_vqa_process_results
+model_specific_prompt_kwargs:
+  default:
+    pre_prompt: ""
+    post_prompt: "\nWhen the provided information is insufficient, respond with 'Unanswerable'.\nAnswer the question using a single word or phrase."
+metadata:
+  - version: 0.0

EAGLE/lmms_eval/tasks/ok_vqa/_generate_config.py

@@ -0,0 +1,25 @@
+import os
+import yaml
+
+splits = ["val2014"]
+tasks = ["vqa"]
+
+if __name__ == "__main__":
+    dump_tasks = []
+    for task in tasks:
+        for split in splits:
+            yaml_dict = {"group": f"ok_vqa", "task": f"ok_vqa_{split}", "include": f"_default_template_{task}_yaml", "test_split": split}
+            if split == "train":
+                yaml_dict.pop("group")
+            else:
+                dump_tasks.append(f"ok_vqa_{split}")
+
+            save_path = f"./ok_vqa_{split}.yaml"
+            print(f"Saving to {save_path}")
+            with open(save_path, "w") as f:
+                yaml.dump(yaml_dict, f, default_flow_style=False, sort_keys=False)
+
+    group_dict = {"group": "ok_vqa", "task": dump_tasks}
+
+    with open("./_ok_vqa.yaml", "w") as f:
+        yaml.dump(group_dict, f, default_flow_style=False, indent=4)

EAGLE/lmms_eval/tasks/ok_vqa/_ok_vqa.yaml

@@ -0,0 +1,3 @@
+group: ok_vqa
+task:
+- ok_vqa_val2014

EAGLE/lmms_eval/tasks/ok_vqa/ok_vqa_val2014.yaml

@@ -0,0 +1,4 @@
+group: ok_vqa
+task: ok_vqa_val2014
+test_split: val2014
+include: _default_template_vqa_yaml

EAGLE/lmms_eval/tasks/ok_vqa/utils.py

@@ -0,0 +1,70 @@
+import re
+import os
+import json
+import yaml
+import pathlib
+import logging
+import datetime
+import statistics
+
+from lmms_eval.tasks._task_utils.file_utils import generate_submission_file
+from lmms_eval.tasks._task_utils.vqa_eval_metric import EvalAIAnswerProcessor
+
+eval_logger = logging.getLogger("lmms-eval")
+
+
+def ok_vqa_doc_to_visual(doc):
+    return [doc["image"].convert("RGB")]
+
+
+def ok_vqa_process_results(doc, result):
+    eval_ai_processor = EvalAIAnswerProcessor()
+    assert len(result) == 1, f"The result should be a list of length 1, but got {len(result)}."
+    resAns = eval_ai_processor(result[0])
+    accuracy = 0
+
+    if "answers" in doc and doc["answers"] is not None:
+        gtAcc = []
+
+        for i in range(len(doc["answers"])):
+            doc["answers"][i] = eval_ai_processor(doc["answers"][i])
+
+        for i in range(len(doc["answers"])):
+            otherGTAns = [doc["answers"][j] for j in range(len(doc["answers"])) if i != j]
+            matchingAns = [item for item in otherGTAns if item == resAns]
+            acc = min(1, float(len(matchingAns)) / 3)
+            gtAcc.append(acc)
+        if gtAcc:
+            accuracy = statistics.mean(gtAcc)
+        else:
+            accuracy = 0
+
+    return {
+        "exact_match": accuracy,
+        "submission": {
+            "image": f"{doc['question_id']}.jpg",
+            "answer": resAns,
+        },
+    }
+
+
+def ok_vqa_doc_to_text(doc, model_specific_prompt_kwargs=None):
+    question = doc["question"]
+    if model_specific_prompt_kwargs is None:
+        model_specific_prompt_kwargs = {}
+    pre_prompt = ""
+    post_prompt = ""
+    if "pre_prompt" in model_specific_prompt_kwargs:
+        pre_prompt = model_specific_prompt_kwargs["pre_prompt"]
+    if "post_prompt" in model_specific_prompt_kwargs:
+        post_prompt = model_specific_prompt_kwargs["post_prompt"]
+    return f"{pre_prompt}{question}{post_prompt}"
+
+
+def ok_vqa_aggreate_submissions(results, args):
+    now_date_time = datetime.datetime.now().strftime("%Y-%m%d-%H%M-%S")
+    file = f"ok_vqa-test-submission-{now_date_time}.json"
+    path = generate_submission_file(file, args)
+    with open(path, "w") as f:
+        json.dump(results, f)
+    print(f"Submission file saved to {path}")

EAGLE/lmms_eval/tasks/olympiadbench/cn_utils.py

@@ -0,0 +1,69 @@
+import os
+import json
+import datetime
+from lmms_eval.tasks.olympiadbench.olympiadbench_evals import OlympiadBenchEvaluator
+from lmms_eval.tasks._task_utils.file_utils import generate_submission_file
+
+import logging
+eval_logger = logging.getLogger("lmms-eval")
+dir_name = os.path.dirname(os.path.abspath(__file__))
+
+olympiadbench_evaluator = OlympiadBenchEvaluator()
+
+def olympiadbench_doc_to_visual(doc):
+    return [image.convert("RGB") for image in doc["images"]]
+
+def olympiadbench_doc_to_text(doc):
+    question = doc["question"]
+    subject = doc["subfield"]
+    mul_ans = doc["is_multiple_answer"]
+    if mul_ans is None:
+        mul_ans = False
+    ans_type = doc["answer_type"]
+    if ans_type == "Need_human_evaluate":
+        ans_type = "proof based"
+
+    pre_prompt = f"以下是中国{subject}竞赛中的解答题。\n"
+
+    post_prompt = ""
+    if not mul_ans:
+        post_prompt += f"答案类型为{ans_type}。\n"
+    else:
+        post_prompt += f"题目有多个答案，答案类型均为{ans_type}。\n"
+    post_prompt += "请根据题目的要求和所提供的信息计算得出答案。解答过程和结果中使用的变量和公式请使用LaTeX格式表示。请在最后以"
+    if not mul_ans:
+        post_prompt += '"所以最终答案是\\boxed{答案}。"\n'
+    else:
+        post_prompt += '"所以最终答案是\\boxed{用英⽂逗号连接的多个答案}。"\n'
+
+    final_question = pre_prompt + question + '\n' + post_prompt
+    return final_question
+
+def olympiadbench_process_results(doc, results):
+    precision = doc["error"]
+    is_proving = "TP" in doc["source"] 
+    if precision is None:
+        precision = 0
+    prediction = results[0].strip()
+
+    if is_proving:
+        return {
+            "submission": prediction
+        }
+    else:
+        prediction = prediction.split("所以最终答案是")[-1]
+        prediction = prediction.replace('"', "").replace("\n", "").replace(" ", "").strip(".").strip("。")
+        accuracy = olympiadbench_evaluator.judge(prediction, doc["final_answer"][0], precision)
+        accuracy = int(accuracy)
+        return {
+            "exact_match": accuracy
+        }
+
+def olympiadbench_aggregate_results(results, args):
+    now_date_time = datetime.datetime.now().strftime("%Y-%m%d-%H%M-%S")
+    submission_file_name = f"olympiadbench-test-cn-submission-{now_date_time}.json"
+    path = generate_submission_file(submission_file_name, args)
+    with open(path, "w") as f:
+        json.dump(results, f, ensure_ascii=False)
+    print(f"Submission file saved to {path}")
+    

EAGLE/lmms_eval/tasks/olympiadbench/en_utils.py

@@ -0,0 +1,69 @@
+import os
+import json
+import datetime
+from lmms_eval.tasks.olympiadbench.olympiadbench_evals import OlympiadBenchEvaluator
+from lmms_eval.tasks._task_utils.file_utils import generate_submission_file
+
+import logging
+eval_logger = logging.getLogger("lmms-eval")
+dir_name = os.path.dirname(os.path.abspath(__file__))
+
+olympiadbench_evaluator = OlympiadBenchEvaluator()
+
+def olympiadbench_doc_to_visual(doc):
+    return [image.convert("RGB") for image in doc["images"]]
+
+def olympiadbench_doc_to_text(doc):
+    question = doc["question"]
+    subject = doc["subfield"]
+    mul_ans = doc["is_multiple_answer"]
+    if mul_ans is None:
+        mul_ans = False
+    ans_type = doc["answer_type"]
+    if ans_type == "Need_human_evaluate":
+        ans_type = "proof based"
+
+    pre_prompt = f"The following is a question from an International {subject} competition.\n"
+
+    post_prompt = ""
+    if not mul_ans:
+        post_prompt += f"The answer of the question should be {ans_type}.\n"
+    else:
+        post_prompt += f"The question has multiple answers, each of them should be {ans_type}.\n"
+    post_prompt += "Please calculate the answer according to the given requirements and the information provided. Please use LaTeX format to represent the variables and formulas used in the solution process and results. Please end your solution with "
+    if not mul_ans:
+        post_prompt += '"So the final answer is \\boxed{answer}."\n'
+    else:
+        post_prompt += 'So the final answer is \\boxed{multiple answers connected with commas}.\n'
+
+    final_question = pre_prompt + question + '\n' + post_prompt
+    return final_question
+
+def olympiadbench_process_results(doc, results):
+    precision = doc["error"]
+    is_proving = "TP" in doc["source"] 
+    if precision is None:
+        precision = 0
+    prediction = results[0].strip()
+
+    if is_proving:
+        return {
+            "submission": prediction
+        }
+    else:
+        prediction = prediction.split("final answer is")[-1]
+        prediction = prediction.replace('"', "").replace("\n", "").replace(" ", "").strip(".").strip("。")
+        accuracy = olympiadbench_evaluator.judge(prediction, doc["final_answer"][0], precision)
+        accuracy = int(accuracy)
+        return {
+            "exact_match": accuracy
+        }
+
+def olympiadbench_aggregate_results(results, args):
+    now_date_time = datetime.datetime.now().strftime("%Y-%m%d-%H%M-%S")
+    submission_file_name = f"olympiadbench-test-en-submission-{now_date_time}.json"
+    path = generate_submission_file(submission_file_name, args)
+    with open(path, "w") as f:
+        json.dump(results, f, ensure_ascii=False)
+    print(f"Submission file saved to {path}")
+    

EAGLE/lmms_eval/tasks/olympiadbench/olympiadbench.yaml

@@ -0,0 +1,6 @@
+group: olympiadbench
+task:
+- olympiadbench_test_en
+- olympiadbench_test_cn
+metadata:
+  - version: 0.0

EAGLE/lmms_eval/tasks/olympiadbench/olympiadbench_evals.py

@@ -0,0 +1,355 @@
+import re
+import sympy as sp
+from sympy import simplify, Eq, sympify, Pow
+from sympy.parsing.latex import parse_latex
+import math
+
+# how to use
+# scorer = OlympiadBenchEvaluator()
+# exp1 = "10^{10^{10^{10}}}"
+# exp2 = "10^{10}"
+# precision = 1e-4
+# res = scorer.judge(exp1, exp2, precision)
+
+class OlympiadBenchEvaluator:
+    def __init__(self):
+        # Map of special symbols to their replacements
+        self.special_signal_map = {
+            "\\left": "",
+            "\\right": "",
+            "∶": ":",
+            "，": ",",
+            "$": "",
+            "\\approx": "=",
+            "\\simeq": "=",
+            "\\sim": "=",
+            "^\\prime": "'",
+            "^{\\prime}": "'",
+            "^\\circ": "",
+            "%": "",
+        }
+        self.pi = parse_latex("\\pi")
+        self.precision = 1e-8  # Default precision for comparison
+
+    def split_by_comma(self, expr: str):
+        # Splits expressions by commas outside of brackets
+        in_bracket_num = 0
+        splitted_expr = []
+        start_idx = 0
+        for i, char in enumerate(expr):
+            if char in ["(", "["]:
+                in_bracket_num += 1
+            elif char in [")", "]"]:
+                in_bracket_num -= 1
+            elif char == "," and in_bracket_num == 0:
+                splitted_expr.append(expr[start_idx:i].strip())
+                start_idx = i + 1
+
+        if start_idx < len(expr):
+            splitted_expr.append(expr[start_idx:].strip())   
+        
+        return splitted_expr
+
+    def trans_plus_minus_sign(self, expr_list: list):
+        # Translates plus-minus signs into separate expressions
+        new_expr_list = []
+        for expr in expr_list:
+            if "\\pm" in expr:
+                new_expr_list.append(expr.replace("\\pm", "+"))
+                new_expr_list.append(expr.replace("\\pm", "-"))
+            else:
+                new_expr_list.append(expr)
+        
+        return new_expr_list
+    
+    def judge(self, expression1, expression2, precision=1e-8):
+        # Judge if two expressions are equal (expression1 is considered as the Ground Truth)
+        # Default precision is a list for supporting multiple expressions
+        precision = precision if isinstance(precision, list) else [precision]
+
+        try:
+            expression1, expression2 = self.preprocess(expression1, expression2)
+        except:
+            return False
+        if expression1 == expression2:
+            # print("Exactly equal")
+            return True
+        
+        # Remove Chinese characters from the string, as answers like "yes" or "no" in Chinese have been considered
+        expression1 = re.sub(r'[\u4e00-\u9fff]+', '', expression1)
+        expression2 = re.sub(r'[\u4e00-\u9fff]+', '', expression2)
+        
+        expression1 = self.split_by_comma(expression1)
+        expression2 = self.split_by_comma(expression2)
+
+        temp_list1 = self.trans_plus_minus_sign(expression1)
+        temp_list2 = self.trans_plus_minus_sign(expression2)
+
+        # Set up a list for allowed errors
+        if len(precision) <= 1:
+            precision = precision * len(temp_list1)
+        
+        if len(temp_list1) != len(temp_list2):
+            return False
+
+        # Check if elements in both lists can be paired and are equal
+        idx = -1
+        while len(temp_list1) != 0:
+            idx = (idx + 1) % len(temp_list1)
+
+            item1 = temp_list1[idx]
+            self.precision = precision[idx]
+
+            for item2 in temp_list2:
+                if self.is_equal(item1, item2):
+                    temp_list1.remove(item1)
+                    temp_list2.remove(item2)
+                    precision.remove(self.precision)
+                    break
+            else:
+                # If no match was found, return False
+                return False
+
+        # If all elements are matched, return True
+        return True
+    
+    def is_interval(self, expr):
+        # Checks if an expression is an interval
+        return expr.startswith(("(", "[")) and expr.endswith((")", "]"))
+
+    def sympy_sub_pi(self, expression_sympy):
+        # Replaces the symbol for pi in sympy expressions with its numerical value
+        return expression_sympy.subs(self.pi, math.pi)
+    
+    def is_equal(self, expression1, expression2):
+        # Default first expression is ground truth. Check if expressions are equal in different aspects
+        if expression1 == expression2 and expression1 != "" and expression2 != "":
+            # print("Equivalent natively")
+            return True
+
+        # First check if both are intervals
+        if self.is_interval(expression1) and self.is_interval(expression2):
+            try:
+                if self.interval_equal(expression1, expression2):
+                    # print("Interval equivalent")
+                    return True
+            except:
+                return False
+
+        # Then check for numerical equality
+        try:
+            if self.numerical_equal(expression1, expression2):
+                # print("Numerically equivalent")
+                return True
+        except:
+            pass
+        
+        # Then check if expressions are mathematically equal
+        try:
+            if self.expression_equal(expression1, expression2) and not ("=" in expression1 and "=" in expression2):
+                # print("Expression equivalent")
+                return True
+        except:
+            pass
+            
+        # Lastly, check for equation equality
+        try:
+            if self.equation_equal(expression1, expression2):
+                # print("Equation equivalent")
+                return True
+        except:
+            pass
+            
+        return False
+
+    def numerical_equal(self, expression1: str, expression2: str, include_percentage: bool = True):
+        # Check if two numerical values are equal within an allowed error range
+        # Includes possible percentage cases
+        reference = float(expression1)
+        prediction = float(expression2)
+        
+        if include_percentage:
+            gt_result = [reference / 100, reference, reference * 100]
+        else:
+            gt_result = [reference]
+        
+        for item in gt_result:
+            if abs(item - prediction) <= self.precision * 1.01:
+                return True
+        return False
+    
+
+    def expression_equal(self, exp1, exp2):
+        # Check if two expressions are mathematically equivalent
+        # Extract expression and use sympy for equivalence checking
+        def extract_expression(expression):
+            if "=" in expression:
+                expression = expression.split("=")[1]
+            return expression.strip()
+        
+        exp1 = extract_expression(exp1)
+        exp2 = extract_expression(exp2)
+
+        expr1_sym = sympify(parse_latex(exp1))
+        expr2_sym = sympify(parse_latex(exp2))
+
+        if expr1_sym == expr2_sym:
+            return True
+        else:
+            expr1_sym = self.sympy_sub_pi(expr1_sym)
+            expr2_sym = self.sympy_sub_pi(expr2_sym)
+
+            if (expr1_sym.has(sp.Symbol) and not expr2_sym.has(sp.Symbol)) or (not expr1_sym.has(sp.Symbol) and expr2_sym.has(sp.Symbol)):
+                return False
+            elif not expr1_sym.has(sp.Symbol) and not expr2_sym.has(sp.Symbol):
+                try:
+                    if not (self.can_compute_power(expr1_sym) and self.can_compute_power(expr2_sym)):
+                        print(f"These two numbers cannot be calculated by the current computer for: \"{str(expr1_sym)}\" and \"{str(expr2_sym)}\"")
+                        return False
+
+                    if abs(expr1_sym.evalf() - expr2_sym.evalf()) <= self.precision * 1.01:
+                        return True
+                    else:
+                        return False
+                except:
+                    return False
+            else:
+                try:
+                    simplified_expr = simplify(expr1_sym - expr2_sym)
+
+                    num_value = simplified_expr.evalf()
+                    
+                    return abs(num_value) < 1e-3
+                except:
+                    return False
+
+    def equation_equal(self, expression1, expression2):
+        # Check if two equations are mathematically equivalent
+        # Simplify equations and use sympy for equivalence checking
+        def simplify_equation(latex_eq):
+            lhs, rhs = latex_eq.split('=')
+
+            lhs_expr = parse_latex(lhs)
+            rhs_expr = parse_latex(rhs)
+
+            equation = Eq(lhs_expr, rhs_expr)
+
+            simplified_eq = simplify(equation.lhs - equation.rhs)
+
+            return simplified_eq
+
+        expr1_sym = simplify_equation(expression1)
+        expr2_sym = simplify_equation(expression2)
+
+        division_result_1 = simplify(expr1_sym / expr2_sym)
+        division_result_2 = simplify(expr2_sym / expr1_sym)
+
+        if (division_result_1.is_Integer and division_result_1 != 0) or (division_result_2.is_Integer and division_result_2 != 0):
+            return True
+        else:
+            return False
+
+    def interval_equal(self, expression1, expression2):
+        # Check if two intervals are mathematically equivalent
+        def compare_two_interval(inter1, inter2):
+            if inter1[0] != inter2[0] or inter1[-1] != inter2[-1]:
+                return False
+            
+            inter1 = inter1.strip('[]()')
+            inter2 = inter2.strip('[]()')
+
+            items_1 = inter1.split(',')
+            items_2 = inter2.split(',')
+
+            for item_1, item_2 in zip(items_1, items_2):
+                if not self.expression_equal(item_1, item_2):
+                    return False
+            return True
+            
+        interval1 = expression1
+        interval2 = expression2
+
+        if interval1 == interval2:
+            return True
+        else:
+            inter_list1 = interval1.split("\\cup")
+            inter_list2 = interval2.split("\\cup")
+            
+            if len(inter_list1) != len(inter_list2):
+                return False
+            else:
+                for inter1, inter2 in zip(inter_list1, inter_list2):
+                    if not compare_two_interval(inter1, inter2):
+                        return False
+                return True
+
+    def preprocess(self, expression1, expression2):
+        # Preprocess expressions to extract and replace special symbols
+        def extract_boxed_content(latex_str):
+            boxed_matches = re.finditer(r'\\boxed{', latex_str)
+            results = ""
+
+            for match in boxed_matches:
+                start_index = match.end()
+                end_index = start_index
+                stack = 1
+
+                while stack > 0 and end_index < len(latex_str):
+                    if latex_str[end_index] == '{':
+                        stack += 1
+                    elif latex_str[end_index] == '}':
+                        stack -= 1
+                    end_index += 1
+
+                if stack == 0:
+                    content = latex_str[start_index:end_index - 1]
+                    results += content + ","
+                else:
+                    raise ValueError("Mismatched braces in LaTeX string.")
+
+            if results == "":
+                last_line_ans = latex_str.strip().split("\n")[-1]
+                dollar_pattern = r"\$(.*?)\$"
+                answers = re.findall(dollar_pattern, last_line_ans)
+
+                if answers:
+                    for ans in answers:
+                        results += ans + ","
+                else:
+                    results = latex_str
+                
+            return results
+        
+        def sepcial_symbol_replace(expression):
+            if "\\in " in expression:
+                expression = expression.split("\\in ")[1]
+            
+            for signal in self.special_signal_map:
+                expression = expression.replace(signal, self.special_signal_map[signal])
+
+            expression = expression.strip("\n$,.:;^_=+`!@#$%^&*~，。")
+
+            pattern = r'\\(?:mathrm|mathbf)\{~?([^}]*)\}'
+            expression = re.sub(pattern, r'\1', expression)
+
+            return expression
+        
+        exp1, exp2 = extract_boxed_content(expression1), extract_boxed_content(expression2)
+        exp1, exp2 = sepcial_symbol_replace(exp1), sepcial_symbol_replace(exp2)
+
+        return exp1, exp2
+    
+    def can_compute_power(self, expr):
+        # Checks if a power expression can be computed
+        if isinstance(expr, Pow):
+            base, exp = expr.as_base_exp()
+            if base.is_number and exp.is_number:
+                MAX_EXP = 1000  # Adjust based on computing environment
+                if abs(exp.evalf()) > MAX_EXP:
+                    return False
+                else:
+                    return True
+            else:
+                return False
+        else:
+            return True  # Not a power expression, can compute

EAGLE/lmms_eval/tasks/olympiadbench/olympiadbench_test_cn.yaml

@@ -0,0 +1,25 @@
+dataset_path: lmms-lab/OlympiadBench
+dataset_kwargs:
+  token: True
+task : "olympiadbench_test_cn"
+test_split: test_cn
+output_type: generate_until
+doc_to_visual: !function cn_utils.olympiadbench_doc_to_visual
+doc_to_text: !function cn_utils.olympiadbench_doc_to_text
+doc_to_target: "answer"
+generation_kwargs:
+  until:
+    - "ASSISTANT:"
+  max_new_tokens: 1024
+  temperature: 0
+  top_p: 0
+  num_beams: 1
+  do_sample: false
+process_results: !function cn_utils.olympiadbench_process_results
+metric_list:
+  - metric: submission
+    aggregation: !function cn_utils.olympiadbench_aggregate_results
+    higher_is_better: true
+  - metric: exact_match
+    aggregation: mean
+    higher_is_better: true

EAGLE/lmms_eval/tasks/olympiadbench/olympiadbench_test_en.yaml

@@ -0,0 +1,25 @@
+dataset_path: lmms-lab/OlympiadBench
+dataset_kwargs:
+  token: True
+task : "olympiadbench_test_en"
+test_split: test_en
+output_type: generate_until
+doc_to_visual: !function en_utils.olympiadbench_doc_to_visual
+doc_to_text: !function en_utils.olympiadbench_doc_to_text
+doc_to_target: "answer"
+generation_kwargs:
+  until:
+    - "ASSISTANT:"
+  max_new_tokens: 1024
+  temperature: 0
+  top_p: 0
+  num_beams: 1
+  do_sample: false
+process_results: !function en_utils.olympiadbench_process_results
+metric_list:
+  - metric: submission
+    aggregation: !function en_utils.olympiadbench_aggregate_results
+    higher_is_better: true
+  - metric: exact_match
+    aggregation: mean
+    higher_is_better: true

EAGLE/lmms_eval/tasks/seedbench/seedbench.yaml

@@ -0,0 +1,28 @@
+dataset_path: lmms-lab/SEED-Bench
+dataset_kwargs:
+  token: True
+task: "seedbench"
+test_split: test
+output_type: generate_until
+doc_to_visual: !function utils.seed_doc_to_visual
+doc_to_text: !function utils.seed_doc_to_text
+doc_to_target: "answer"
+generation_kwargs:
+  until:
+    - "ASSISTANT:"
+  image_aspect_ratio: original
+# The return value of process_results will be used by metrics
+process_results: !function utils.seed_process_result
+# Note that the metric name can be either a registed metric function (such as the case for GQA) or a key name returned by process_results
+metric_list:
+  - metric: seed_image
+    aggregation: !function utils.seed_aggregation_result
+    higher_is_better: true
+  - metric: seed_video
+    aggregation: !function utils.seed_aggregation_result
+    higher_is_better: true
+  - metric: seed_all
+    aggregation: !function utils.seed_aggregation_result
+    higher_is_better: true
+metadata:
+  - version: 0.0

EAGLE/lmms_eval/tasks/seedbench/seedbench_ppl.yaml

@@ -0,0 +1,15 @@
+dataset_path: lmms-lab/SEED-Bench
+dataset_kwargs:
+  token: True
+task: "seedbench_ppl"
+test_split: test
+output_type: multiple_choice
+doc_to_visual: !function utils.seed_doc_to_visual
+doc_to_text: !function utils.seed_doc_to_text_mc
+doc_to_choice : !function utils.seed_doc_to_choice
+doc_to_target: !function utils.seed_doc_to_mc_target
+# Note that the metric name can be either a registed metric function (such as the case for GQA) or a key name returned by process_results
+metric_list:
+  - metric: acc
+metadata:
+  - version: 0.0

EAGLE/lmms_eval/tasks/seedbench/utils.py

@@ -0,0 +1,60 @@
+import json
+
+
+def seed_doc_to_visual(doc):
+    return [image.convert("RGB") for image in doc["image"]]
+
+
+def seed_doc_to_text(doc):
+    question = doc["question"]
+    question += "\n" + f"A. {doc['choice_a']}\n"
+    question += f"B. {doc['choice_b']}\n"
+    question += f"C. {doc['choice_c']}\n"
+    question += f"D. {doc['choice_d']}"
+    return f"{question}\nAnswer with the option's letter from the given choices directly."
+
+
+def seed_process_result(doc, result):
+    pred = result[0].strip()
+    if len(pred) > 1:
+        pred = pred[0]
+    answer = doc["answer"]
+    data_type = doc["data_type"]
+
+    return {f"seed_{data_type}": {"pred": pred, "answer": answer, "question_id": doc["question_id"]}, f"seed_all": {"pred": pred, "answer": answer, "question_id": doc["question_id"]}}
+
+
+def seed_aggregation_result(results):
+    total_count = 0
+    total_correct = 0
+    for result in results:
+        if result["pred"] == result["answer"]:
+            total_correct += 1
+        total_count += 1
+    return total_correct / total_count
+
+
+def seed_aggregation_result_all(results):
+    score = seed_aggregation_result(results)
+    stored_results = []
+    for result in results:
+        stored_results.append({"question_id": result["question_id"], "prediction": result["pred"]})
+    with open("./seed_submission.json", "w") as f:
+        json.dump(stored_results, f, indent=4)
+    print("Storing files for seed_submission ...")
+
+    return score
+
+
+def seed_doc_to_text_mc(doc):
+    question = doc["question"]
+    return f"{question} Answer :"
+
+
+def seed_doc_to_choice(doc):
+    return [doc["choice_a"], doc["choice_b"], doc["choice_c"], doc["choice_d"]]
+
+
+def seed_doc_to_mc_target(doc):
+    answer2choice = {"A": "choice_a", "B": "choice_b", "C": "choice_c", "D": "choice_d"}
+    return doc[answer2choice[doc["answer"]]]

EAGLE/lmms_eval/tasks/textvqa/_default_template_textvqa_yaml

@@ -0,0 +1,17 @@
+dataset_path: lmms-lab/textvqa
+output_type: generate_until
+doc_to_visual: !function utils.textvqa_doc_to_visual
+doc_to_text: !function utils.textvqa_doc_to_text
+doc_to_target: "answer"
+generation_kwargs:
+  until:
+    - "ASSISTANT:"
+process_results: !function utils.textvqa_process_results
+model_specific_prompt_kwargs:
+  default:
+    pre_prompt: ""
+    post_prompt: "\nAnswer the question using a single word or phrase."
+    ocr: true
+  qwen_vl:
+    pre_prompt: ""
+    post_prompt: " Answer:"

EAGLE/lmms_eval/tasks/textvqa/_textvqa.yaml

@@ -0,0 +1,4 @@
+group: textvqa
+task:
+- textvqa_val
+- textvqa_test

EAGLE/lmms_eval/tasks/textvqa/textvqa_test.yaml

@@ -0,0 +1,7 @@
+task: textvqa_test
+test_split: test
+metric_list:
+  - metric: submission
+    aggregation: !function utils.textvqa_aggreate_submissions
+    higher_is_better: true
+include: _default_template_textvqa_yaml

EAGLE/lmms_eval/tasks/textvqa/textvqa_val.yaml

@@ -0,0 +1,12 @@
+task: textvqa_val
+test_split: validation
+metric_list:
+  - metric: exact_match
+    aggregation: mean
+    higher_is_better: true
+    ignore_case: true
+    ignore_punctuation: true
+  - metric: submission
+    aggregation: !function utils.textvqa_aggreate_submissions
+    higher_is_better: true
+include: _default_template_textvqa_yaml

EAGLE/lmms_eval/tasks/textvqa/utils.py

@@ -0,0 +1,68 @@
+import re
+import os
+import json
+import yaml
+import pathlib
+import logging
+import datetime
+import statistics
+
+from lmms_eval.tasks._task_utils.vqa_eval_metric import EvalAIAnswerProcessor
+from lmms_eval.tasks._task_utils.file_utils import generate_submission_file
+
+eval_logger = logging.getLogger("lmms-eval")
+
+
+def textvqa_doc_to_visual(doc):
+    return [doc["image"].convert("RGB")]
+
+
+def textvqa_process_results(doc, result):
+    eval_ai_processor = EvalAIAnswerProcessor()
+    assert len(result) == 1, f"The result should be a list of length 1, but got {len(result)}."
+    resAns = eval_ai_processor(result[0])
+    accuracy = 0
+
+    if "answers" in doc and doc["answers"] is not None:
+        gtAcc = []
+
+        for i in range(len(doc["answers"])):
+            doc["answers"][i] = eval_ai_processor(doc["answers"][i])
+
+        for i in range(len(doc["answers"])):
+            otherGTAns = [doc["answers"][j] for j in range(len(doc["answers"])) if i != j]
+            matchingAns = [item for item in otherGTAns if item == resAns]
+            acc = min(1, float(len(matchingAns)) / 3)
+            gtAcc.append(acc)
+        accuracy = statistics.mean(gtAcc)
+
+    return {
+        "exact_match": accuracy,
+        "submission": {
+            "question_id": doc["question_id"],
+            "answer": resAns,
+        },
+    }
+
+
+def textvqa_doc_to_text(doc, model_specific_prompt_kwargs=None):
+    pre_prompt = ""
+    post_post = ""
+    ocr_ref = ""
+    if model_specific_prompt_kwargs:
+        if "pre_prompt" in model_specific_prompt_kwargs:
+            pre_prompt = model_specific_prompt_kwargs["pre_prompt"]
+        if "post_prompt" in model_specific_prompt_kwargs:
+            post_prompt = model_specific_prompt_kwargs["post_prompt"]
+        if "ocr" in model_specific_prompt_kwargs and model_specific_prompt_kwargs["ocr"]:
+            ocr_ref = f"\nReference OCR token: {', '.join(doc['ocr_tokens'])}"
+    return f"{pre_prompt}{doc['question'].capitalize()}{ocr_ref}{post_prompt}"
+
+
+def textvqa_aggreate_submissions(results, args):
+    now_date_time = datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S")
+    path = generate_submission_file(f"textvqa_submission_{now_date_time}.json", args)
+    with open(path, "w") as f:
+        json.dump(results, f)
+    # print(f"Submission file saved to {path}")
+    eval_logger.info(f"Submission file saved to {path}")

EAGLE/lmms_eval/tasks/vizwiz_vqa/_default_template_vqa_yaml

@@ -0,0 +1,15 @@
+dataset_path: lmms-lab/VizWiz-VQA
+output_type: generate_until
+doc_to_visual: !function utils.vizwiz_vqa_doc_to_visual
+doc_to_text: !function utils.vizwiz_vqa_doc_to_text
+doc_to_target: "answer"
+generation_kwargs:
+  until:
+    - "ASSISTANT:"
+metadata:
+  - version: 0.0
+model_specific_prompt_kwargs:
+  default:
+    pre_prompt: ""
+    post_prompt: "\nWhen the provided information is insufficient, respond with 'Unanswerable'.\nAnswer the question using a single word or phrase."
+process_results: !function utils.vizwiz_vqa_process_results

EAGLE/lmms_eval/tasks/vizwiz_vqa/_generate_config.py

@@ -0,0 +1,25 @@
+import os
+import yaml
+
+splits = ["val", "test"]
+tasks = ["vqa"]
+
+if __name__ == "__main__":
+    dump_tasks = []
+    for task in tasks:
+        for split in splits:
+            yaml_dict = {"group": f"vizwiz_{task}", "task": f"vizwiz_{task}_{split}", "include": f"_default_template_{task}_yaml", "test_split": split}
+            if split == "train":
+                yaml_dict.pop("group")
+            else:
+                dump_tasks.append(f"vizwiz_{task}_{split}")
+
+            save_path = f"./vizwiz_{task}_{split}.yaml"
+            print(f"Saving to {save_path}")
+            with open(save_path, "w") as f:
+                yaml.dump(yaml_dict, f, default_flow_style=False, sort_keys=False)
+
+    group_dict = {"group": "vizwiz_vqa", "task": dump_tasks}
+
+    with open("./_vizwiz_vqa.yaml", "w") as f:
+        yaml.dump(group_dict, f, default_flow_style=False, indent=4)

EAGLE/lmms_eval/tasks/vizwiz_vqa/_vizwiz_vqa.yaml

@@ -0,0 +1,4 @@
+group: vizwiz_vqa
+task:
+- vizwiz_vqa_val
+- vizwiz_vqa_test

EAGLE/lmms_eval/tasks/vizwiz_vqa/utils.py

@@ -0,0 +1,70 @@
+import re
+import os
+import json
+import yaml
+import pathlib
+import logging
+import datetime
+import statistics
+
+from lmms_eval.tasks._task_utils.file_utils import generate_submission_file
+from lmms_eval.tasks._task_utils.vqa_eval_metric import EvalAIAnswerProcessor
+
+eval_logger = logging.getLogger("lmms-eval")
+
+
+def vizwiz_vqa_doc_to_visual(doc):
+    return [doc["image"].convert("RGB")]
+
+
+def vizwiz_vqa_process_results(doc, result):
+    eval_ai_processor = EvalAIAnswerProcessor()
+    assert len(result) == 1, f"The result should be a list of length 1, but got {len(result)}."
+    resAns = eval_ai_processor(result[0])
+    accuracy = 0
+
+    if "answers" in doc and doc["answers"] is not None:
+        gtAcc = []
+
+        for i in range(len(doc["answers"])):
+            doc["answers"][i] = eval_ai_processor(doc["answers"][i])
+
+        for i in range(len(doc["answers"])):
+            otherGTAns = [doc["answers"][j] for j in range(len(doc["answers"])) if i != j]
+            matchingAns = [item for item in otherGTAns if item == resAns]
+            acc = min(1, float(len(matchingAns)) / 3)
+            gtAcc.append(acc)
+        if gtAcc:
+            accuracy = statistics.mean(gtAcc)
+        else:
+            accuracy = 0
+
+    return {
+        "exact_match": accuracy,
+        "submission": {
+            "image": f"{doc['question_id']}.jpg",
+            "answer": resAns,
+        },
+    }
+
+
+def vizwiz_vqa_doc_to_text(doc, model_specific_prompt_kwargs=None):
+    if model_specific_prompt_kwargs is None:
+        model_specific_prompt_kwargs = {}
+    pre_prompt = ""
+    post_prompt = ""
+    if "pre_prompt" in model_specific_prompt_kwargs:
+        pre_prompt = model_specific_prompt_kwargs["pre_prompt"]
+    if "post_prompt" in model_specific_prompt_kwargs:
+        post_prompt = model_specific_prompt_kwargs["post_prompt"]
+    text = f"{pre_prompt}{doc['question'].capitalize()}{post_prompt}"
+    return text
+
+
+def vizwiz_vqa_aggreate_submissions(results, args):
+    now_date_time = datetime.datetime.now().strftime("%Y-%m%d-%H%M-%S")
+    submission_file_name = f"vizwiz_vqa-test-submission-{now_date_time}.json"
+    path = generate_submission_file(submission_file_name, args)
+    with open(path, "w") as f:
+        json.dump(results, f)
+    print(f"Submission file saved to {path}")

EAGLE/lmms_eval/tasks/vizwiz_vqa/vizwiz_vqa_test.yaml

@@ -0,0 +1,14 @@
+group: vizwiz_vqa
+task: vizwiz_vqa_test
+test_split: test
+include: _default_template_vqa_yaml
+process_results: !function utils.vizwiz_vqa_process_results
+metric_list:
+  # - metric: exact_match
+  #   aggregation: mean
+  #   higher_is_better: true
+  #   ignore_case: true
+  #   ignore_punctuation: true
+  - metric: submission
+    aggregation: !function utils.vizwiz_vqa_aggreate_submissions
+    higher_is_better: true

EAGLE/lmms_eval/tasks/vizwiz_vqa/vizwiz_vqa_val.yaml

@@ -0,0 +1,13 @@
+group: vizwiz_vqa
+task: vizwiz_vqa_val
+test_split: val
+include: _default_template_vqa_yaml
+metric_list:
+  - metric: exact_match
+    aggregation: mean
+    higher_is_better: true
+    ignore_case: true
+    ignore_punctuation: true
+  # - metric: submission
+  #   aggregation: !function utils.vizwiz_vqa_aggreate_submissions
+  #   higher_is_better: true

EAGLE/lmms_eval/tasks/vqav2/_default_template_vqav2_yaml

@@ -0,0 +1,15 @@
+dataset_path: lmms-lab/VQAv2
+dataset_kwargs:
+  token: True
+output_type: generate_until
+doc_to_visual: !function utils.vqav2_doc_to_visual
+doc_to_text: !function utils.vqav2_doc_to_text
+doc_to_target: "answer"
+generation_kwargs:
+  max_new_tokens: 16
+metadata:
+  - version: 0.0
+model_specific_prompt_kwargs:
+  default:
+    pre_prompt: ""
+    post_prompt: "\nAnswer the question using a single word or phrase."

EAGLE/lmms_eval/tasks/vqav2/_vqav2.yaml

@@ -0,0 +1,4 @@
+group: vqav2
+task:
+- vqav2_val
+- vqav2_test

EAGLE/lmms_eval/tasks/vqav2/utils.py

@@ -0,0 +1,89 @@
+import re
+import os
+import json
+import logging
+import datetime
+import statistics
+
+import lmms_eval.tasks._task_utils.file_utils as file_utils
+
+from lmms_eval.tasks._task_utils.vqa_eval_metric import EvalAIAnswerProcessor
+
+
+eval_logger = logging.getLogger("lmms-eval")
+
+
+def vqav2_doc_to_visual(doc):
+    return [doc["image"].convert("RGB")]
+
+
+def vqav2_process_results(doc, result):
+    eval_ai_processor = EvalAIAnswerProcessor()
+    assert len(result) == 1, f"The result should be a list of length 1, but got {len(result)}."
+    resAns = eval_ai_processor(result[0])
+    accuracy = 0
+
+    if "answers" in doc and doc["answers"] is not None:
+        for ansDic in doc["answers"]:
+            ansDic["answer"] = ansDic["answer"].replace("\n", " ")
+            ansDic["answer"] = ansDic["answer"].replace("\t", " ")
+            ansDic["answer"] = ansDic["answer"].strip()
+        gtAcc = []
+        gtAnswers = [ans["answer"] for ans in doc["answers"]]
+
+        if len(set(gtAnswers)) > 1:
+            for ansDic in doc["answers"]:
+                ansDic["answer"] = eval_ai_processor.process_punctuation(ansDic["answer"])
+                ansDic["answer"] = eval_ai_processor.process_digit_article(ansDic["answer"])
+            resAns = eval_ai_processor.process_punctuation(resAns)
+            resAns = eval_ai_processor.process_digit_article(resAns)
+
+        for gtAnsDatum in doc["answers"]:
+            otherGTAns = [item for item in doc["answers"] if item != gtAnsDatum]
+            matchingAns = [item for item in otherGTAns if item["answer"] == resAns]
+            acc = min(1, float(len(matchingAns)) / 3)
+            gtAcc.append(acc)
+        accuracy = statistics.mean(gtAcc)
+
+    return {
+        "exact_match": accuracy,
+        "submission": {
+            "question_id": doc["question_id"],
+            "answer": resAns,
+        },
+    }
+
+
+def vqav2_process_results_test(doc, result):
+    res = vqav2_process_results(doc, result)
+    return {
+        "submission": res["submission"],
+    }
+
+
+def vqav2_process_results_val(doc, result):
+    res = vqav2_process_results(doc, result)
+    return {
+        "exact_match": res["exact_match"],
+    }
+
+
+def vqav2_doc_to_text(doc, model_specific_prompt_kwargs=None):
+    if model_specific_prompt_kwargs is None:
+        model_specific_prompt_kwargs = {}
+    pre_prompt = ""
+    post_prompt = ""
+    if "pre_prompt" in model_specific_prompt_kwargs:
+        pre_prompt = model_specific_prompt_kwargs["pre_prompt"]
+    if "post_prompt" in model_specific_prompt_kwargs:
+        post_prompt = model_specific_prompt_kwargs["post_prompt"]
+    return f"{pre_prompt}{doc['question']}{post_prompt}"
+
+
+def vqav2_aggreate_submissions(results, args):
+    now_date_time = datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S")
+    submission_file_name = f"vqav2-test-submission-{now_date_time}.json"
+    path = file_utils.generate_submission_file(submission_file_name, args)
+    with open(path, "w") as f:
+        json.dump(results, f)
+    eval_logger.info(f"Submission file saved to {path}")

EAGLE/lmms_eval/tasks/vqav2/vqav2_test.yaml

@@ -0,0 +1,8 @@
+task: "vqav2_test"
+include: _default_template_vqav2_yaml
+test_split: test
+metric_list:
+  - metric: submission
+    aggregation: !function utils.vqav2_aggreate_submissions
+    higher_is_better: true
+process_results: !function utils.vqav2_process_results_test