import pandas as pd import os import pprint as pp # import requests # ========================== # Define some column names MODEL_SIZE_COL_NAME = 'Size(B)' # ========================= from datasets import DATASETS HF_TOKEN = os.environ.get("HF_TOKEN") BASE_COLS = ["Rank", "Models", MODEL_SIZE_COL_NAME, "Data Source"] TASKS_V1 = ["V1-Overall", "I-CLS", "I-QA", "I-RET", "I-VG"] COLUMN_NAMES = BASE_COLS + TASKS_V1 DATA_TITLE_TYPE = ['number', 'markdown', 'str', 'markdown'] + \ ['number'] * len(TASKS_V1) LEADERBOARD_INTRODUCTION = """ # 📊 **MMEB LEADERBOARD (VLM2Vec)** ## Introduction We introduce **Massive Multimodal Embedding Benchmark (MMEB)**, a novel comprehensive benchmark for evaluating omni-modality embedding models across text, image, video, audio, visual document, and agent-centric retrieval scenarios. **MMEB-V1** includes 36 datasets spanning four image-text meta-task categories: classification, visual question answering, retrieval, visual grounding. **MMEB-V2** expands the evaluation scope to include five new tasks: - four video-based tasks: Video Retrieval, Moment Retrieval, Video Classification, and Video Question Answering - one visual documents task: Visual Document Retrieval. **MMEB-V3** further extends to a fuller modality setting by adding three major new evaluation categories: - Audio Tasks: audio classification, cross-modal audio retrieval, and audio temporal grounding. - Text Retrieval: instruction-following retrieval, reasoning retrieval, long-context retrieval, multi-condition retrieval, and general text retrieval. - Agent Tasks: tool retrieval, GUI control, and agent memory retrieval.
""" ANNOUNCEMENT = """ """ LEADERBOARD_INFO = f""" ## Dataset Overview This is the dictionary of all datasets used in our code. Please make sure all datasets' scores are included in your submission. \n ```python {pp.pformat(DATASETS)} ``` """ CITATION_BUTTON_LABEL = "Copy the following snippet to cite these results" CITATION_BUTTON_TEXT = r"""@article{jiang2024vlm2vec, title={VLM2Vec: Training Vision-Language Models for Massive Multimodal Embedding Tasks}, author={Jiang, Ziyan and Meng, Rui and Yang, Xinyi and Yavuz, Semih and Zhou, Yingbo and Chen, Wenhu}, journal={arXiv preprint arXiv:2410.05160}, year={2024} }""" SUBMIT_INTRODUCTION = """# Submit on MMEB Leaderboard Introduction \n ## Please follow the guidelines in order to submit successfully. \n 1. **Step 1️⃣:** Please refer to the [**GitHub page**](https://github.com/TIGER-AI-Lab/VLM2Vec) for detailed instructions about evaluating your model. \n - If you want to submit to a specific modality leaderboard, such as MMEB Image, only run your model on the corresponding datasets and ignore the remaining. - However, your model will still be shown on all leaderboards and might have a lower rank since missing datasets will be assigned a 0. \n 2. **Step 2️⃣:** After running the evaluation pipelines, please use the provided script **(e.g., [report_score_v3.py](https://github.com/TIGER-AI-Lab/VLM2Vec/blob/main/experiments/report_score_v3.py))** to generate the final score sheet. - Adjust your model's configurations in the script before running it - Note the "model size" field is digits-only and is in Billions (B) (ex., "8" for 8 billions, "0.5" for 500 millions). - If possible, please also add a contact method in case we want to reach you in the future. 3. **Step 3️⃣:** Finally, create a pull request and upload the generated JSON file to the ***scores*** folder. - If directly using web UI: Go to the [scores folder](https://huggingface.co/spaces/TIGER-Lab/MMEB-Leaderboard/upload/main/scores), select "Upload file" and upload your JSON files. - If by git command line: refer to the [PR documentation](https://huggingface.co/docs/hub/repositories-pull-requests-discussions#pull-requests-advanced-usage). - Submit the PR and leave comments if any. We will then review and update the leaderboard accordingly.\n - To delete or modify your submission, submit a new PR with the updated file.\n\n ## 🐞 Bug reporting and feedback If you encounter any issues or have feedback for improvement regarding the leaderboard, please report them in [Discussion](https://huggingface.co/spaces/TIGER-Lab/MMEB-Leaderboard/discussions).\n If you cannot reach us via above methods, email us at **m7su@uwaterloo.ca**. ## Appendix 1: Example valid score sheet format ⬇️: \n ```json { "metadata": { "model_name": "", "url": "" or null, "model_size": or null, "contact": xxx@xxxxx.com ... ... }, "metrics": { "image": { "ImageNet-1K": { "hit@1": 0.5, "ndcg@1": 0.5, ... ... }, "N24News": { ... ... }, ... ... }, "video": { ... ... }, ... ... } } ``` """ def create_hyperlinked_names(df): def convert_url(url, model_name): return f'{model_name}' if url else model_name def add_link_to_model_name(row): row['Models'] = convert_url(row['URL'], row['Models']) return row df = df.copy() df = df.apply(add_link_to_model_name, axis=1) return df # def fetch_data(file: str) -> pd.DataFrame: # # fetch the leaderboard data from remote # if file is None: # raise ValueError("URL Not Provided") # url = f"https://huggingface.co/spaces/TIGER-Lab/MMEB/resolve/main/{file}" # print(f"Fetching data from {url}") # response = requests.get(url) # if response.status_code != 200: # raise requests.HTTPError(f"Failed to fetch data: HTTP status code {response.status_code}") # return pd.read_json(io.StringIO(response.text), orient='records', lines=True) def get_df(file="results.jsonl"): df = pd.read_json(file, orient='records', lines=True) df[MODEL_SIZE_COL_NAME] = df['Model Size(B)'].apply(process_model_size) for task in TASKS_V1: if df[task].isnull().any(): df[task] = df[task].apply(lambda score: '-' if pd.isna(score) else score) df = df.sort_values(by=['V1-Overall'], ascending=False) df = create_hyperlinked_names(df) df['Rank'] = range(1, len(df) + 1) return df def refresh_data(): df = get_df() return df[COLUMN_NAMES] def search_and_filter_models(df, query, min_size, max_size): filtered_df = df.copy() if query: filtered_df = filtered_df[filtered_df['Models'].str.contains(query, case=False, na=False)] size_mask = filtered_df[MODEL_SIZE_COL_NAME].apply(lambda x: (min_size <= 1000.0 <= max_size) if x == 'unknown' else (min_size <= x <= max_size)) filtered_df = filtered_df[size_mask] return filtered_df[COLUMN_NAMES] def search_models(df, query): if query: return df[df['Models'].str.contains(query, case=False, na=False)] return df def get_size_range(df): sizes = df[MODEL_SIZE_COL_NAME].apply(lambda x: 0.0 if x == 'unknown' else x) if (sizes == 0.0).all(): return 0.0, 1000.0 return float(sizes.min()), float(sizes.max()) def process_model_size(size): if pd.isna(size) or size == 'unk': return 'unknown' try: val = float(size) return round(val, 3) except (ValueError, TypeError): return 'unknown' def filter_columns_by_tasks(df, selected_tasks=None): if selected_tasks is None or len(selected_tasks) == 0: return df[COLUMN_NAMES] base_columns = ['Models', MODEL_SIZE_COL_NAME, 'Data Source', 'Overall'] selected_columns = base_columns + selected_tasks available_columns = [col for col in selected_columns if col in df.columns] return df[available_columns]