utils.py

import pandas as pd
import os
import pprint as pp
# import requests

from datasets import DATASETS

HF_TOKEN = os.environ.get("HF_TOKEN")

BASE_COLS = ["Rank", "Models", "Model Size(B)", "Data Source"]
TASKS_V1 = ["V1-Overall", "I-CLS", "I-QA", "I-RET", "I-VG"]
COLUMN_NAMES = BASE_COLS + TASKS_V1

DATA_TITLE_TYPE = ['number', 'markdown', 'str', 'markdown'] + \
                    ['number'] * len(TASKS_V1)

LEADERBOARD_INTRODUCTION = """
# 📊 **MMEB LEADERBOARD (VLM2Vec)**

## Introduction
We introduce **Massive Multimodal Embedding Benchmark (MMEB)**, a novel comprehensive benchmark for evaluating omni-modality embedding models across text, image, video, audio, visual document, and agent-centric retrieval scenarios.  

**MMEB-V1** includes 36 datasets spanning four image-text meta-task categories: classification, visual question answering, retrieval, visual grounding.

**MMEB-V2** expands the evaluation scope to include five new tasks: 
- four video-based tasks: Video Retrieval, Moment Retrieval, Video Classification, and Video Question Answering 
- one task focused on visual documents: Visual Document Retrieval. 

**MMEB-V3** further extends to a fuller modality setting by adding three major new evaluation categories:
- Audio Tasks: audio classification, cross-modal audio retrieval, and audio temporal grounding.
- Text Retrieval: instruction-following retrieval, reasoning retrieval, long-context retrieval, multi-condition retrieval, and general text retrieval.
- Agent Tasks: tool retrieval, GUI control, and agent memory retrieval.

<div style="display:inline-flex; flex-wrap:wrap; gap:6px; align-items:center; margin:8px 0;">
<a target="_blank" href="https://tiger-ai-lab.github.io/VLM2Vec/">
<img style="height:17pt" src="https://img.shields.io/badge/-🌐%20Website-red?style=flat"></a>
<a target="_blank" href="https://github.com/TIGER-AI-Lab/VLM2Vec">
<img style="height:17pt" src="https://img.shields.io/badge/-MMEB--V3%20Code-green?style=flat&logo=github"></a>
<a target="_blank" href="https://huggingface.co/datasets/VLM2Vec/MMEB-V3">
<img style="height:17pt" src="https://img.shields.io/badge/-🤗%20Dataset(MMEB--V3)-red?style=flat"></a>
<a target="_blank" href="https://arxiv.org/abs/2604.23321">
<img style="height:17pt" src="https://img.shields.io/badge/-V3 Paper-black?style=flat&logo=arxiv"></a>
<a target="_blank" href="https://arxiv.org/abs/2507.04590">
<img style="height:17pt" src="https://img.shields.io/badge/-V2 Paper-black?style=flat&logo=arxiv"></a>
<a target="_blank" href="https://arxiv.org/abs/2410.05160">
<img style="height:17pt" src="https://img.shields.io/badge/-V1 Paper-black?style=flat&logo=arxiv"></a>
<a target="_blank" href="https://huggingface.co/datasets/TIGER-Lab/MMEB-V2">
<img style="height:17pt" src="https://img.shields.io/badge/-🤗%20Dataset(V2)-red?style=flat"></a>
<a target="_blank" href="https://huggingface.co/datasets/TIGER-Lab/MMEB-eval">
<img style="height:17pt" src="https://img.shields.io/badge/-🤗%20Dataset(V1)-red?style=flat"></a>
<a target="_blank" href="https://huggingface.co/VLM2Vec">
<img style="height:17pt" src="https://img.shields.io/badge/-🤗%20Models-red?style=flat"></a>
<a target="_blank" href="https://huggingface.co/spaces/TIGER-Lab/MMEB-Leaderboard">
<img style="height:17pt" src="https://img.shields.io/badge/-🤗%20Leaderboard-red?style=flat"></a>
</div>
"""

ANNOUNCEMENT = """
<ul>
<li>[2026-07] MMEB-V3 Leaderboard is now live!</li>
<li>[2026-07] MMEB-V3 is accepted to COLM 2026!</li>
<li>[2026-04] MMEB-V3 released!</li>
<li>[2026-01] VLM2Vec/MMEB-V2 is accepted to TMLR 2026!</li>
<li>[2025-06] VLM2Vec/MMEB-V2 released!</li>
</ul>"""

LEADERBOARD_INFO = f"""
## Dataset Overview
This is the dictionary of all datasets used in our code. Please make sure all datasets' scores are included in your submission. \n
```python
{pp.pformat(DATASETS)}
```
"""

CITATION_BUTTON_LABEL = "Copy the following snippet to cite these results"
CITATION_BUTTON_TEXT = r"""@article{jiang2024vlm2vec,
  title={VLM2Vec: Training Vision-Language Models for Massive Multimodal Embedding Tasks},
  author={Jiang, Ziyan and Meng, Rui and Yang, Xinyi and Yavuz, Semih and Zhou, Yingbo and Chen, Wenhu},
  journal={arXiv preprint arXiv:2410.05160},
  year={2024}
}"""

SUBMIT_INTRODUCTION = """# Submit on MMEB Leaderboard Introduction \n
## Please follow the guidelines in order to submit successfully. \n
1. **Step  1️⃣:** Please refer to the [**GitHub page**](https://github.com/TIGER-AI-Lab/VLM2Vec) for detailed instructions about evaluating your model. \n
2. **Step  2️⃣:** After running the evaluation pipelines, please use the provided script **(e.g., [report_score_v2.py](https://github.com/TIGER-AI-Lab/VLM2Vec/blob/main/experiments/report_score_v2.py))** to generate the final score sheet. (Use [report_score_v3.py](https://github.com/TIGER-AI-Lab/VLM2Vec/blob/main/experiments/report_score_v3.py) for v3 submission).
    - Adjust your model's configurations in the script before running it
        - Note the "model size" field is digits-only and is in Billions (B), so please convert it if yours is in different units/formats (e.x., "8" for 8 billion, "0.5" for 500 million).
        - If possible, please also add a contact method in case we want to reach you in the future
3. **Step  3️⃣:** Finally, create a pull request and upload the generated JSON file to the ***scores*** folder. 
    - If directly using web UI:
        - Go to the [scores folder](https://huggingface.co/spaces/TIGER-Lab/MMEB-Leaderboard/upload/main/scores)
        - Select "Upload file" and upload your JSON files.
    - If by git command line: refer to the [PR documentation](https://huggingface.co/docs/hub/repositories-pull-requests-discussions#pull-requests-advanced-usage).
    - Submit the PR and leave any comments if any. We will then review and update the leaderboard accordingly.\n
    - To delete or modify your submission, submit a new PR with the updated file.\n\n

## 🐞 Bug reporting and feedback
If you encounter any issues or have improvement feedback regarding the leaderboard, please report them in Discussion.\n
If you cannot reach us via above methods, email us at **m7su@uwaterloo.ca**. 

## Appendix 1: Example: valid score sheet format ⬇️: \n
```json
{
    "metadata": {
        "model_name": "<Model Name>",
        "url": "<Model URL>" or null,
        "model_size": <Model Size> or null,
        "contact": xxx@xxxxx.com
        ... ...
    },
    "metrics": {
        "image": {
            "ImageNet-1K": {
                "hit@1": 0.5,
                "ndcg@1": 0.5,
                ... ...
            }, 
            "N24News": {
                ... ...
            }, 
            ... ...
        }, 
        "video": {
            ... ...
        },
        ... ...
    }
}
```
## ⚠️ Special Instructions for submitting to MMEB Image (Previously MMEB-V1) Leaderboard
We understand that some researchers want to exclusively submit to the Image leaderboard, but unfortunately our current leaderboard cannot exclude your model from other modalities' leaderboards. 
To do so, run the 36 image datasets only and simply ignore other datasets.
The leaderboard will automatically assign a 0 to the missing datasets and your model will be shown on all leaderboards, and might have a lower rank. \n
"""

def create_hyperlinked_names(df):
    def convert_url(url, model_name):
        return f'<a href="{url}">{model_name}</a>' if url else model_name

    def add_link_to_model_name(row):
        row['Models'] = convert_url(row['URL'], row['Models'])
        return row
    
    df = df.copy()
    df = df.apply(add_link_to_model_name, axis=1)
    return df

# def fetch_data(file: str) -> pd.DataFrame:
#     # fetch the leaderboard data from remote
#     if file is None:
#         raise ValueError("URL Not Provided")
#     url = f"https://huggingface.co/spaces/TIGER-Lab/MMEB/resolve/main/{file}"
#     print(f"Fetching data from {url}")
#     response = requests.get(url)
#     if response.status_code != 200:
#         raise requests.HTTPError(f"Failed to fetch data: HTTP status code {response.status_code}")
#     return pd.read_json(io.StringIO(response.text), orient='records', lines=True)

def get_df(file="results.jsonl"):
    df = pd.read_json(file, orient='records', lines=True)
    df['Model Size(B)'] = df['Model Size(B)'].apply(process_model_size)
    for task in TASKS_V1:
        if df[task].isnull().any():
            df[task] = df[task].apply(lambda score: '-' if pd.isna(score) else score)
    df = df.sort_values(by=['V1-Overall'], ascending=False)
    df = create_hyperlinked_names(df)
    df['Rank'] = range(1, len(df) + 1)
    return df

def refresh_data():
    df = get_df()
    return df[COLUMN_NAMES]

def search_and_filter_models(df, query, min_size, max_size):
    filtered_df = df.copy()
    
    if query:
        filtered_df = filtered_df[filtered_df['Models'].str.contains(query, case=False, na=False)]

    size_mask = filtered_df['Model Size(B)'].apply(lambda x: 
        (min_size <= 1000.0 <= max_size) if x == 'unknown' 
        else (min_size <= x <= max_size))
    
    filtered_df = filtered_df[size_mask]
    
    return filtered_df[COLUMN_NAMES]


def search_models(df, query):
    if query:
        return df[df['Models'].str.contains(query, case=False, na=False)]
    return df

def get_size_range(df):
    sizes = df['Model Size(B)'].apply(lambda x: 0.0 if x == 'unknown' else x)
    if (sizes == 0.0).all():
        return 0.0, 1000.0
    return float(sizes.min()), float(sizes.max())


def process_model_size(size):
    if pd.isna(size) or size == 'unk':
        return 'unknown'
    try:
        val = float(size)
        return round(val, 3)
    except (ValueError, TypeError):
        return 'unknown'

def filter_columns_by_tasks(df, selected_tasks=None):
    if selected_tasks is None or len(selected_tasks) == 0:
        return df[COLUMN_NAMES]
    
    base_columns = ['Models', 'Model Size(B)', 'Data Source', 'Overall']
    selected_columns = base_columns + selected_tasks
    
    available_columns = [col for col in selected_columns if col in df.columns]
    return df[available_columns]