update upload battle

arena_elo/elo_rating/clean_battle_data.py

@@ -251,8 +251,6 @@ def clean_battle_data(
                 continue
         
         
-        question_id = row["states"][0]["conv_id"]
-
         ip = row["ip"]
         if ip not in all_ips:
             all_ips[ip] = {"ip": ip, "count": 0, "sanitized_id": len(all_ips)}
@@ -266,13 +264,40 @@ def clean_battle_data(
             ct_banned += 1
             print(f"User {user_id} is banned")
             continue
-
+        required_keys_each_task = {
+            "image_editing": ["source_prompt", "target_prompt", "instruct_prompt"],
+            "t2i_generation": ["prompt"],
+            "video_generation": ["prompt"]
+        }
+        
+        model_a_inputs = row["states"][0].copy()
+        # pop conv_id and model_name
+        model_a_inputs.pop("conv_id")
+        model_a_inputs.pop("model_name")
+        model_b_inputs = row["states"][1].copy()
+        model_b_inputs.pop("conv_id")
+        model_b_inputs.pop("model_name")
+        for key in model_a_inputs:
+            if not (key in model_b_inputs and model_a_inputs[key] == model_b_inputs[key]):
+                print(f"Inconsistent inputs: {model_a_inputs} vs {model_b_inputs}")
+                ct_invalid += 1
+                continue
+        if mode == "conv_release":
+            if any(key not in model_a_inputs for key in required_keys_each_task[task_name]):
+                print(f"Missing required keys: {model_a_inputs}, {required_keys_each_task[task_name]}")
+                ct_invalid += 1
+                continue
+            
+        inputs = model_a_inputs
         # Save the results
         battles.append(
             dict(
-                question_id=question_id,
+                model_a_conv_id=row["states"][0]["conv_id"],
+                model_b_conv_id=row["states"][1]["conv_id"],
+                inputs=inputs,
                 model_a=models[0],
                 model_b=models[1],
+                vote_type=row["type"],
                 winner=convert_type[row["type"]],
                 judge=f"arena_user_{user_id}",
                 anony=anony,
@@ -330,14 +355,14 @@ if __name__ == "__main__":
     ).strftime("%Y%m%d")
 
     if args.mode == "simple":
-        for x in battles:
-            for key in [
-                "conversation_a",
-                "conversation_b",
-                "question_id",
-            ]:
-                if key in x:
-                    del x[key]
+        # for x in battles:
+        #     for key in [
+        #         "conversation_a",
+        #         "conversation_b",
+        #         "question_id",
+        #     ]:
+        #         if key in x:
+        #             del x[key]
         print("Samples:")
         for i in range(min(4, len(battles))):
             print(battles[i])

arena_elo/elo_rating/upload_battle_data.py

@@ -2,60 +2,36 @@ import fire
 import json
 import os
 import datasets
+import random
 import datetime
 from pathlib import Path
 from datetime import datetime
 from PIL import Image
 
 datasets.config.DEFAULT_MAX_BATCH_SIZE = 500
-def create_hf_dataset(data_file: str, split="test"):
-    hf_dataset = datasets.Dataset.from_list(
-        data_file,
-        features=datasets.Features(
-            {
-                "question_id": datasets.Value("string"),
-                "model": datasets.Value("string"),
-                "conversation": [
-                    {
-                        "role": datasets.Value("string"),
-                        "content": datasets.Value("string"),
-                    }
-                ],
-                "language": datasets.Value("string"),
-                "image": datasets.Image(),
-                "turn": datasets.Value("int32"),
-            }
-        ),
-        split=split,
-    )
-    return hf_dataset
 
-def create_hf_battle_dataset(data_file: str, split="test"):
-    hf_dataset = datasets.Dataset.from_list(
-        data_file,
-        features=datasets.Features(
+def create_hf_battle_dataset(data_file: str, split="test", task_type="t2i_generation"):
+    if task_type == "t2i_generation":
+        features = datasets.Features(
             {
-                "question_id": datasets.Value("string"),
-                "model_a": datasets.Value("string"),
-                "model_b": datasets.Value("string"),
-                "conversation_a": [
-                    {
-                        "role": datasets.Value("string"),
-                        "content": datasets.Value("string"),
-                    }
-                ],
-                "conversation_b": [
-                    {
-                        "role": datasets.Value("string"),
-                        "content": datasets.Value("string"),
-                    }
-                ],
-                "language": datasets.Value("string"),
-                "image": datasets.Image(),
-                "turn": datasets.Value("int32"),
+                "index": datasets.Value("int32"),
+                "tstamp": datasets.Value("int32"),
+                "prompt": datasets.Value("string"),
+                "left_model": datasets.Value("string"),
+                "left_image": datasets.Image(),
+                "right_model": datasets.Value("string"),
+                "right_image": datasets.Image(),
+                "vote_type": datasets.Value("string"),
+                "winner": datasets.Value("string"),
                 "anony": datasets.Value("bool"),
+                "judge": datasets.Value("string"),
             }
-        ),
+        )
+    else:
+        raise ValueError(f"Task type {task_type} not supported")
+    hf_dataset = datasets.Dataset.from_list(
+        data_file,
+        features=features,
         split=split,
     )
     return hf_dataset
@@ -81,106 +57,102 @@ def get_date_from_time_stamp(unix_timestamp: int):
 def load_battle_image(battle, log_dir):
     image_path = Path(log_dir) / f"{get_date_from_time_stamp(battle['tstamp'])}-convinput_images" / f"input_image_{battle['question_id']}.png"
     return load_image(image_path)
+
+def find_media_path(conv_id, task_type, log_dir):
+    media_directory_map = {
+        "t2i_generation": "images/generation",
+        "image_edition": "images/edition",
+        "text2video": "videos/generation"
+    }
+    if task_type == "t2i_generation":
+        media_path = Path(log_dir) / media_directory_map[task_type] / f"{conv_id}.jpg"
+    else:
+        raise ValueError(f"Task type {task_type} not supported")
+    return media_path
     
 
 def main(
-    data_file: str = "./results/latest/clean_battle_conv.json",
-    repo_id: str = "DongfuTingle/wildvision-bench",
-    log_dir: str = os.getenv("LOGDIR", "./vision-arena-logs/"),
-    mode="battle",
-    token = os.environ.get("HUGGINGFACE_TOKEN", None)
+    task_type='t2i_generation',
+    # data_file: str = "./results/latest/clean_battle_conv.json",
+    data_file: str = None,
+    repo_id: str = "DongfuJiang/GenAI-Arena-human-eval",
+    log_dir: str = os.getenv("LOGDIR", "../GenAI-Arena-hf-logs/vote_log"),
+    config_name='battle',
+    split='test',
+    token = os.environ.get("HUGGINGFACE_TOKEN", None),
+    seed=42,
 ):
+    if data_file is None:
+        data_file = f"./results/latest/clean_battle_{task_type}.json"
+    if not os.path.exists(data_file):
+        raise ValueError(f"Data file {data_file} does not exist")
     with open(data_file, "r") as f:
         data = json.load(f)
     
+    # add index according to the tsamp
+    if seed is not None:
+        random.seed(seed)
+    # data = random.sample(data, 50 * 7)
     
     
-    has_image_stats = {
-        "has_image": 0,
-        "no_image": 0,
+    data = sorted(data, key=lambda x: x['tstamp'])
+    required_keys_each_task = {
+        "image_editing": ["source_prompt", "target_prompt", "instruct_prompt"],
+        "t2i_generation": ["prompt"],
+        "video_generation": ["prompt"]
     }
-    if mode == "keep_bad_only":
-        # anony only
-        data = [d for d in data if d["anony"]]
-        
-        new_data = []
-        for battle in data:
-            image = load_battle_image(battle, log_dir)
-            if image is None:
-                has_image_stats["no_image"] += 1
-                # we don't keep the data without image
-                continue
-            has_image_stats["has_image"] += 1
-            
-            if battle["winner"] in ["model_a", "model_b"]:
-                if battle["winner"] == "model_a":
-                    worse_model = "model_b"
-                    worse_conv = "conversation_b"
-                if battle["winner"] == "model_b":
-                    worse_model = "model_a"
-                    worse_conv = "conversation_a"
-                    
-                new_data.append({
-                    "question_id": battle["question_id"],
-                    "model": battle[worse_model],
-                    "conversation": battle[worse_conv],
-                    "language": battle["language"],
-                    "image": image,
-                    "turn": battle["turn"],
-                })
-            elif battle["winner"] == "tie (bothbad)":
-                
-                new_data.append({
-                    "question_id": battle["question_id"],
-                    "model": battle["model_a"],
-                    "conversation": battle["conversation_a"],
-                    "language": battle["language"],
-                    "image": image,
-                    "turn": battle["turn"],
-                })
-
-                new_data.append({
-                    "question_id": battle["question_id"],
-                    "model": battle["model_b"],
-                    "conversation": battle["conversation_b"],
-                    "language": battle["language"],
-                    "image": image,
-                    "turn": battle["turn"],
-                })
-                
-        split = "test"
-        hf_dataset = create_hf_dataset(new_data, "test")
+    valid_data = []
+    for i, battle in enumerate(data):
+        if any(key not in battle['inputs'] for key in required_keys_each_task[task_type]):
+            # print(f"Skipping battle {i} due to missing keys")
+            continue
+        valid_data.append(battle)
+    data = valid_data
     
-    elif mode == "battle":
-        new_data = []
+    for i, battle in enumerate(data):
+        battle['index'] = i
+    
+
+    new_data = []
+    if task_type == 't2i_generation':
         for battle in data:
-            image = load_battle_image(battle, log_dir)
-            if image is None:
-                has_image_stats["no_image"] += 1
+            prompt = battle['inputs']['prompt']
+            model_a = battle['model_a']
+            model_b = battle['model_b']
+            model_a_conv_id = battle['model_a_conv_id']
+            model_b_conv_id = battle['model_b_conv_id']
+            tstamp = battle['tstamp']
+            vote_type = battle['vote_type']
+            left_image_path = find_media_path(model_a_conv_id, task_type, log_dir)
+            right_image_path = find_media_path(model_b_conv_id, task_type, log_dir)
+            left_image = load_image(left_image_path)
+            right_image = load_image(right_image_path)
+            if left_image is None or right_image is None:
+                print(f"Skipping battle {battle['index']} due to missing images")
                 continue
-            has_image_stats["has_image"] += 1
             new_data.append({
-                "question_id": battle["question_id"],
-                "model_a": battle["model_a"],
-                "model_b": battle["model_b"],
-                "conversation_a": battle["conversation_a"],
-                "conversation_b": battle["conversation_b"],
-                "language": battle["language"],
-                "image": image,
-                "turn": battle["turn"],
-                "anony": battle["anony"],
+                "index": battle['index'],
+                "tstamp": tstamp,
+                "prompt": prompt,
+                "left_model": model_a,
+                "left_image": left_image,
+                "right_model": model_b,
+                "right_image": right_image,
+                "vote_type": vote_type,
+                "winner": battle['winner'],
+                "anony": battle['anony'],
+                "judge": battle['judge'],
             })
         split = "test"
-        hf_dataset = create_hf_battle_dataset(new_data, "test")
+        hf_dataset = create_hf_battle_dataset(new_data, split, task_type)
     else:
-        raise ValueError(f"Invalid mode: {mode}")
+        raise ValueError(f"Task type {task_type} not supported")
 
-    print(f"Stats: {has_image_stats}")
     print(hf_dataset)
     print(f"Uploading to part {repo_id}:{split}...")
     hf_dataset.push_to_hub(
         repo_id=repo_id,
-        config_name=mode,
+        config_name=config_name,
         split=split,
         token=token,
         commit_message=f"Add vision-arena {split} dataset",

arena_elo/results/20240829/elo_results_t2i_generation.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5ef671f77ad5445c22d89e41197fb29a719cd60ec5371370b7d9d3b8369b2c03
+size 88249

arena_elo/results/20240829/t2i_generation_leaderboard.csv

@@ -0,0 +1,18 @@
+key,Model,Arena Elo rating (anony),Arena Elo rating (full),License,Organization,Link
+PlayGround V2.5,PlayGround V2.5,1125.9647113219344,1126.8179449346965,Playground v2.5 Community License,Playground,https://huggingface.co/playgroundai/playground-v2.5-1024px-aesthetic
+FLUX.1-dev,FLUX.1-dev,1122.977921149468,1141.8553966548204,flux-1-dev-non-commercial-license (other),Black Forest Labs,https://huggingface.co/docs/diffusers/main/en/api/pipelines/flux
+FLUX.1-schnell,FLUX.1-schnell,1083.197039124354,1078.930126686188,Apache-2.0,Black Forest Labs,https://huggingface.co/docs/diffusers/main/en/api/pipelines/flux
+PlayGround V2,PlayGround V2,1072.626941667019,1070.726547878426,Playground v2 Community License,Playground,https://huggingface.co/playgroundai/playground-v2-1024px-aesthetic
+Kolors,Kolors,1052.908730084688,1050.9850967380553,Apache-2.0,Kwai Kolors,https://huggingface.co/Kwai-Kolors/Kolors
+HunyuanDiT,HunyuanDiT,1041.9179059135952,1029.5490717475357,tencent-hunyuan-community,Tencent,https://github.com/Tencent/HunyuanDiT
+StableCascade,StableCascade,1037.9168175776274,1040.2341802192325,stable-cascade-nc-community (other),Stability AI,https://fal.ai/models/stable-cascade/api
+SDXL-Lightning,SDXL-Lightning,1023.8917986436493,1027.1783912113383,openrail++,ByteDance,https://huggingface.co/ByteDance/SDXL-Lightning
+PixArtAlpha,PixArtAlpha,1022.2807006610543,1011.6090234692596,openrail++,PixArt-alpha,https://huggingface.co/PixArt-alpha/PixArt-XL-2-1024-MS
+PixArtSigma,PixArtSigma,1019.027621124307,1017.9548990662921,openrail++,PixArt-alpha,https://github.com/PixArt-alpha/PixArt-sigma
+AuraFlow,AuraFlow,1005.6783096127664,1001.6148483504423,Apache-2.0,Fal.AI,https://huggingface.co/fal/AuraFlow
+SD3,SD3,991.5967272859457,987.1118479592413,stabilityai-nc-research-community,Stability AI,https://huggingface.co/blog/sd3
+SDXL,SDXL,964.6926541747703,964.4225091224126,openrail++,Stability AI,https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0
+SDXLTurbo,SDXLTurbo,912.792093681317,909.3024088309655,sai-nc-community (other),Stability AI,https://huggingface.co/stabilityai/sdxl-turbo
+LCM(v1.5/XL),LCM(v1.5/XL),905.2873322742623,897.684769464938,openrail++,Latent Consistency,https://fal.ai/models/fast-lcm-diffusion-turbo
+OpenJourney,OpenJourney,827.6690928608571,821.5243787424323,creativeml-openrail-m,PromptHero,https://huggingface.co/prompthero/openjourney
+LCM,LCM,789.5736028423817,802.4526781048648,MIT License,Tsinghua University,https://huggingface.co/SimianLuo/LCM_Dreamshaper_v7

arena_elo/results/latest/elo_results_t2i_generation.pkl

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:588311a3f3a0a82a99b6c93e4ddc49a5e9dbb3062ddadc8efe1975de696a0199
-size 88248
+oid sha256:5ef671f77ad5445c22d89e41197fb29a719cd60ec5371370b7d9d3b8369b2c03
+size 88249

arena_elo/results/latest/t2i_generation_leaderboard.csv

@@ -1,18 +1,18 @@
 key,Model,Arena Elo rating (anony),Arena Elo rating (full),License,Organization,Link
-PlayGround V2.5,PlayGround V2.5,1126.2671033011611,1127.144846385844,Playground v2.5 Community License,Playground,https://huggingface.co/playgroundai/playground-v2.5-1024px-aesthetic
-FLUX.1-dev,FLUX.1-dev,1122.9582048577554,1141.6536600935985,flux-1-dev-non-commercial-license (other),Black Forest Labs,https://huggingface.co/docs/diffusers/main/en/api/pipelines/flux
-FLUX.1-schnell,FLUX.1-schnell,1083.0152074641153,1078.8262638227134,Apache-2.0,Black Forest Labs,https://huggingface.co/docs/diffusers/main/en/api/pipelines/flux
-PlayGround V2,PlayGround V2,1073.0339489627977,1071.1452200571498,Playground v2 Community License,Playground,https://huggingface.co/playgroundai/playground-v2-1024px-aesthetic
-Kolors,Kolors,1052.8872305800394,1051.0132314632656,Apache-2.0,Kwai Kolors,https://huggingface.co/Kwai-Kolors/Kolors
-HunyuanDiT,HunyuanDiT,1042.095914504853,1029.7357908718288,tencent-hunyuan-community,Tencent,https://github.com/Tencent/HunyuanDiT
-StableCascade,StableCascade,1038.2021066856098,1040.5631483675481,stable-cascade-nc-community (other),Stability AI,https://fal.ai/models/stable-cascade/api
-SDXL-Lightning,SDXL-Lightning,1024.2153507509286,1027.5400610626618,openrail++,ByteDance,https://huggingface.co/ByteDance/SDXL-Lightning
-PixArtAlpha,PixArtAlpha,1022.9514312384921,1012.2288089547184,openrail++,PixArt-alpha,https://huggingface.co/PixArt-alpha/PixArt-XL-2-1024-MS
-PixArtSigma,PixArtSigma,1019.3453069221349,1018.2702682745402,openrail++,PixArt-alpha,https://github.com/PixArt-alpha/PixArt-sigma
-AuraFlow,AuraFlow,1001.228997074127,997.114469839989,Apache-2.0,Fal.AI,https://huggingface.co/fal/AuraFlow
-SD3,SD3,991.7812080130284,987.2707845693864,stabilityai-nc-research-community,Stability AI,https://huggingface.co/blog/sd3
-SDXL,SDXL,965.1076294757463,964.8000746525106,openrail++,Stability AI,https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0
-SDXLTurbo,SDXLTurbo,913.1686128053395,909.6861549501493,sai-nc-community (other),Stability AI,https://huggingface.co/stabilityai/sdxl-turbo
-LCM(v1.5/XL),LCM(v1.5/XL),905.6902615871095,898.0724427034747,openrail++,Latent Consistency,https://fal.ai/models/fast-lcm-diffusion-turbo
-OpenJourney,OpenJourney,828.0685532627626,821.9169694977911,creativeml-openrail-m,PromptHero,https://huggingface.co/prompthero/openjourney
-LCM,LCM,789.9829325139982,802.8893988540598,MIT License,Tsinghua University,https://huggingface.co/SimianLuo/LCM_Dreamshaper_v7
+PlayGround V2.5,PlayGround V2.5,1125.9647113219344,1126.8179449346965,Playground v2.5 Community License,Playground,https://huggingface.co/playgroundai/playground-v2.5-1024px-aesthetic
+FLUX.1-dev,FLUX.1-dev,1122.977921149468,1141.8553966548204,flux-1-dev-non-commercial-license (other),Black Forest Labs,https://huggingface.co/docs/diffusers/main/en/api/pipelines/flux
+FLUX.1-schnell,FLUX.1-schnell,1083.197039124354,1078.930126686188,Apache-2.0,Black Forest Labs,https://huggingface.co/docs/diffusers/main/en/api/pipelines/flux
+PlayGround V2,PlayGround V2,1072.626941667019,1070.726547878426,Playground v2 Community License,Playground,https://huggingface.co/playgroundai/playground-v2-1024px-aesthetic
+Kolors,Kolors,1052.908730084688,1050.9850967380553,Apache-2.0,Kwai Kolors,https://huggingface.co/Kwai-Kolors/Kolors
+HunyuanDiT,HunyuanDiT,1041.9179059135952,1029.5490717475357,tencent-hunyuan-community,Tencent,https://github.com/Tencent/HunyuanDiT
+StableCascade,StableCascade,1037.9168175776274,1040.2341802192325,stable-cascade-nc-community (other),Stability AI,https://fal.ai/models/stable-cascade/api
+SDXL-Lightning,SDXL-Lightning,1023.8917986436493,1027.1783912113383,openrail++,ByteDance,https://huggingface.co/ByteDance/SDXL-Lightning
+PixArtAlpha,PixArtAlpha,1022.2807006610543,1011.6090234692596,openrail++,PixArt-alpha,https://huggingface.co/PixArt-alpha/PixArt-XL-2-1024-MS
+PixArtSigma,PixArtSigma,1019.027621124307,1017.9548990662921,openrail++,PixArt-alpha,https://github.com/PixArt-alpha/PixArt-sigma
+AuraFlow,AuraFlow,1005.6783096127664,1001.6148483504423,Apache-2.0,Fal.AI,https://huggingface.co/fal/AuraFlow
+SD3,SD3,991.5967272859457,987.1118479592413,stabilityai-nc-research-community,Stability AI,https://huggingface.co/blog/sd3
+SDXL,SDXL,964.6926541747703,964.4225091224126,openrail++,Stability AI,https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0
+SDXLTurbo,SDXLTurbo,912.792093681317,909.3024088309655,sai-nc-community (other),Stability AI,https://huggingface.co/stabilityai/sdxl-turbo
+LCM(v1.5/XL),LCM(v1.5/XL),905.2873322742623,897.684769464938,openrail++,Latent Consistency,https://fal.ai/models/fast-lcm-diffusion-turbo
+OpenJourney,OpenJourney,827.6690928608571,821.5243787424323,creativeml-openrail-m,PromptHero,https://huggingface.co/prompthero/openjourney
+LCM,LCM,789.5736028423817,802.4526781048648,MIT License,Tsinghua University,https://huggingface.co/SimianLuo/LCM_Dreamshaper_v7