datasets.py

def sum_lol(lol):
    assert isinstance(lol, list) and all(isinstance(i, list) for i in lol), f"Input should be a list of lists, got {type(lol)}"
    total = []
    for sublist in lol:
        total.extend(sublist)
    return total

SCORE_BASE_DIR = "scores"
META_DATA = ["model_name", "model_size", "url", "report_generated_date"]
DATASETS = {
    "text": {
        "FollowIR": ['core17-instructions', 'news21-instructions', 'robust04-instructions'],
        "R2MED": ['Bioinformatics', 'Biology', 'IIYi-Clinical', 'MedQA-Diag', 'MedXpertQA-Exam', 'Medical-Sciences', 'PMC-Clinical', 'PMC-Treatment'],
        "InfoSearch": ['Audience-v1', 'Clarity-v1', 'Format-v1', 'Language-v1', 'Length-v1', 'Source-v1'],
        "BRIGHT": ['aops', 'biology', 'earth_science', 'economics', 'leetcode', 'pony', 'psychology', 'robotics', 'stackoverflow', 'sustainable_living', 'theoremqa_questions', 'theoremqa_theorems'],
        "LongEmbed": ['2wikimqa', 'narrativeqa', 'needle', 'passkey', 'qmsum', 'summ_screen_fd'],
        "MultiConIR": ['Books', 'Legal Document', 'Medical Case', 'Movies', 'People'],
        "NanoBEIR": ['NanoArguAna', 'NanoClimateFEVER', 'NanoDBPedia', 'NanoFEVER', 'NanoFiQA2018', 'NanoHotpotQA', 'NanoMSMARCO', 'NanoNFCorpus', 'NanoNQ', 'NanoQuoraRetrieval', 'NanoSCIDOCS', 'NanoSciFact', 'NanoTouche2020']
    }, 
    "image": {
        "I-CLS": ['VOC2007', 'N24News', 'SUN397', 'ObjectNet', 'Country211', 'Place365', 'ImageNet-1K', 'HatefulMemes', 'ImageNet-A', 'ImageNet-R'], 
        "I-QA": ['OK-VQA', 'A-OKVQA', 'DocVQA', 'InfographicsVQA', 'ChartQA', 'Visual7W', 'ScienceQA', 'GQA', 'TextVQA', 'VizWiz'], 
        "I-RET": ['VisDial', 'CIRR', 'VisualNews_t2i', 'VisualNews_i2t', 'MSCOCO_t2i', 'MSCOCO_i2t', 'NIGHTS', 'WebQA', 'FashionIQ', 'Wiki-SS-NQ', 'OVEN', 'EDIS'],
        "I-VG": ['MSCOCO', 'RefCOCO', 'RefCOCO-Matching', 'Visual7W-Pointing']
        }, 
    "visdoc": {
        "ViDoRe-V1": ['ViDoRe_arxivqa', 'ViDoRe_docvqa', 'ViDoRe_infovqa', 'ViDoRe_tabfquad', 'ViDoRe_tatdqa', 'ViDoRe_shiftproject', 'ViDoRe_syntheticDocQA_artificial_intelligence', 'ViDoRe_syntheticDocQA_energy', 'ViDoRe_syntheticDocQA_government_reports', 'ViDoRe_syntheticDocQA_healthcare_industry'], 
        "ViDoRe-V2": ["ViDoRe_esg_reports_human_labeled_v2","ViDoRe_biomedical_lectures_v2_multilingual", "ViDoRe_economics_reports_v2_multilingual", "ViDoRe_esg_reports_v2_multilingual"], # "ViDoRe_biomedical_lectures_v2", "ViDoRe_economics_reports_v2", "ViDoRe_esg_reports_v2"
        "VisRAG": ['VisRAG_ArxivQA', 'VisRAG_ChartQA', 'VisRAG_MP-DocVQA', 'VisRAG_SlideVQA', 'VisRAG_InfoVQA', 'VisRAG_PlotQA'], 
        "VisDoc-OOD": ['ViDoSeek-page-fixed', 'ViDoSeek-doc', 'MMLongBench-page-fixed', 'MMLongBench-doc']
        }, 
    "video": {
        "V-CLS": ['K700', 'UCF101', 'HMDB51', 'SmthSmthV2', 'Breakfast'], 
        "V-QA": ['Video-MME', 'MVBench', 'NExTQA', 'EgoSchema', 'ActivityNetQA'], 
        "V-RET": ['MSR-VTT', 'MSVD', 'DiDeMo', 'VATEX', 'YouCook2'], 
        "V-MRET": ['QVHighlight', 'Charades-STA', 'MomentSeeker']
        }, 
    "audio": {
        "A-CLS": ["SpeechCommands", "UrbanSound8K", "ESC-50", "NSynth", "CREMA-D"], 
        "A-RET": ['Clotho', 'SoundDescs', 'TUTSound', 'TUTSound(hard)', 'AVE', 'SpeechCOCO']
        },
    "agent": {
        "Tool": ['apibank', 'apigen', 'mnms', 'reversechain', 'rotbench', 't-eval-dialog', 't-eval-step', 'taskbench-daily', 'toolace', 'toolbench', 'toolemu', 'tooleyes', 'toollens', 'ultratool', 'autotools-food', 'autotools-music', 'autotools-weather', 'restgpt-spotify', 'restgpt-tmdb',
                  'craft-math-algebra', 'craft-tabmwp', 'craft-vqa', 'gorilla-huggingface', 'gorilla-pytorch', 'gorilla-tensor', 'toolink', 
                  'appbench', 'gpt4tools', 'gta', 'taskbench-huggingface', 'taskbench-multimedia', 'metatool', 'tool-be-honest', 'toolalpaca', 'toolbench-sam'],
        "GUI": ['GAE-GUIAct_q2t', 'GAE-GUIAct_q2s', 'GAE-GUIAct_s2s', 'GAE-GUIAct_t2s', 'GAE-Mind2Web_q2t', 'GAE-Mind2Web_q2s', 'GAE-Mind2Web_s2s', 'GAE-Mind2Web_t2s'],
        "Memory": ["REALTALK", "KnowMeBench", "PeerQA", "DeepPlanning"]
    }
}
ALL_DATASETS_SPLITS = {k: sum_lol(list(v.values())) for k, v in DATASETS.items()}
ALL_DATASETS = sum_lol(list(ALL_DATASETS_SPLITS.values()))
MODALITIES = list(DATASETS.keys())
SPECIAL_METRICS = {
    '__default__': 'hit@1',
}