update

.gitattributes

@@ -195,3 +195,39 @@ results/claude-3-sonnet-20240229.jpg filter=lfs diff=lfs merge=lfs -text
 results/claude-3-sonnet-20240229.pkl filter=lfs diff=lfs merge=lfs -text
 results/deepseek-llm-67b-chat.jpg filter=lfs diff=lfs merge=lfs -text
 results-vision/claude-3-opus-20240229.csv filter=lfs diff=lfs merge=lfs -text
+results_qwen/CodeLlama-70b-Instruct-hf.csv filter=lfs diff=lfs merge=lfs -text
+results_qwen/claude-3-opus-20240229.png filter=lfs diff=lfs merge=lfs -text
+results_qwen/gpt-4-0125-preview.png filter=lfs diff=lfs merge=lfs -text
+results_qwen/gpt-4-turbo-2024-04-09.jpg filter=lfs diff=lfs merge=lfs -text
+results_qwen/claude-3-haiku-20240307.csv filter=lfs diff=lfs merge=lfs -text
+results_qwen/gemma-7b-it.pkl filter=lfs diff=lfs merge=lfs -text
+results_qwen/gpt-4-0125-preview.csv filter=lfs diff=lfs merge=lfs -text
+results_qwen/gpt-4-0125-preview.pkl filter=lfs diff=lfs merge=lfs -text
+results_qwen/CodeLlama-70b-Instruct-hf.jpg filter=lfs diff=lfs merge=lfs -text
+results_qwen/Mixtral-8x7B-Instruct-v0.1.jpg filter=lfs diff=lfs merge=lfs -text
+results_qwen/deepseek-llm-67b-chat.pkl filter=lfs diff=lfs merge=lfs -text
+results_qwen/gemma-7b-it.jpg filter=lfs diff=lfs merge=lfs -text
+results_qwen/gemma-7b-it.png filter=lfs diff=lfs merge=lfs -text
+results_qwen/gpt-3.5-turbo-0125.pkl filter=lfs diff=lfs merge=lfs -text
+results_qwen/Mixtral-8x7B-Instruct-v0.1.pkl filter=lfs diff=lfs merge=lfs -text
+results_qwen/gpt-4-0125-preview.jpg filter=lfs diff=lfs merge=lfs -text
+results_qwen/gpt-4-turbo-2024-04-09.csv filter=lfs diff=lfs merge=lfs -text
+results_qwen/gpt-4-turbo-2024-04-09.png filter=lfs diff=lfs merge=lfs -text
+results_qwen/Mixtral-8x7B-Instruct-v0.1.csv filter=lfs diff=lfs merge=lfs -text
+results_qwen/Mixtral-8x7B-Instruct-v0.1.png filter=lfs diff=lfs merge=lfs -text
+results_qwen/claude-3-haiku-20240307.pkl filter=lfs diff=lfs merge=lfs -text
+results_qwen/claude-3-haiku-20240307.png filter=lfs diff=lfs merge=lfs -text
+results_qwen/claude-3-opus-20240229.jpg filter=lfs diff=lfs merge=lfs -text
+results_qwen/deepseek-llm-67b-chat.csv filter=lfs diff=lfs merge=lfs -text
+results_qwen/CodeLlama-70b-Instruct-hf.pkl filter=lfs diff=lfs merge=lfs -text
+results_qwen/deepseek-llm-67b-chat.jpg filter=lfs diff=lfs merge=lfs -text
+results_qwen/gpt-3.5-turbo-0125.csv filter=lfs diff=lfs merge=lfs -text
+results_qwen/claude-3-opus-20240229.csv filter=lfs diff=lfs merge=lfs -text
+results_qwen/claude-3-opus-20240229.pkl filter=lfs diff=lfs merge=lfs -text
+results_qwen/gemma-7b-it.csv filter=lfs diff=lfs merge=lfs -text
+results_qwen/gpt-3.5-turbo-0125.jpg filter=lfs diff=lfs merge=lfs -text
+results_qwen/gpt-3.5-turbo-0125.png filter=lfs diff=lfs merge=lfs -text
+results_qwen/gpt-4-turbo-2024-04-09.pkl filter=lfs diff=lfs merge=lfs -text
+results_qwen/CodeLlama-70b-Instruct-hf.png filter=lfs diff=lfs merge=lfs -text
+results_qwen/claude-3-haiku-20240307.jpg filter=lfs diff=lfs merge=lfs -text
+results_qwen/deepseek-llm-67b-chat.png filter=lfs diff=lfs merge=lfs -text

app.py

@@ -15,7 +15,8 @@ import pandas as pd
 
 
 # Load text benchmark results
-csv_results = glob("results/*.pkl")
+noncot_results = glob("results/*.pkl")
+noncot_results_qwen = glob("results_qwen/*.pkl")
 # Load vision benchmark results
 vision_results = glob("results-vision/*.pkl")
 # Load CoT text benchmark results
@@ -36,7 +37,7 @@ def load_data(files, model_type):
 
 
 # Load and label all data
-data = load_data(csv_results, "Text Only")
+data = load_data(noncot_results, "Text Only")
 vision_data = load_data(vision_results, "Vision")
 cot_text_data = load_data(cot_text_results, "CoT Text Only")
 # cot_vision_data = load_data(cot_vision_results, "CoT Vision")
@@ -61,7 +62,7 @@ text_only_filtered_raw_cot = None
 
 
 # Load the csv files into a dict with keys being name of the file and values being the data
-data = {file: pd.read_pickle(file) for file in csv_results}
+data = {file: pd.read_pickle(file) for file in noncot_results}
 # Load the vision files into a dict
 vision_data = {file: pd.read_pickle(file) for file in vision_results}
 # Load the CoT text files into a dict
@@ -69,6 +70,8 @@ cot_text_data = {file: pd.read_pickle(file) for file in cot_text_results}
 # Load the CoT vision files into a dict
 # cot_vision_data = {file: pd.read_pickle(file) for file in cot_vision_results}
 
+data_qwen = {file: pd.read_pickle(file) for file in noncot_results_qwen}
+
 
 intersection_df = pd.read_pickle(
     "./intersection_results/gpt-3.5-judge-by_Qwen_5times_intersection_subset_1.pkl"
@@ -124,12 +127,15 @@ def process_data(data):
 
 # Process all data
 text_data_for_df = process_data(data)
+text_data_for_df_qwen = process_data(data_qwen)
+
 vision_data_for_df = process_data(vision_data)
 cot_text_data_for_df = process_data(cot_text_data)
 # cot_vision_data_for_df = process_data(cot_vision_data)
 
 # Create DataFrames
 accuracy_df = pd.DataFrame(text_data_for_df, columns=column_names)
+accuracy_df_qwen = pd.DataFrame(text_data_for_df_qwen, columns=column_names)
 vision_accuracy_df = pd.DataFrame(vision_data_for_df, columns=column_names)
 cot_text_accuracy_df = pd.DataFrame(cot_text_data_for_df, columns=column_names)
 # cot_vision_accuracy_df = pd.DataFrame(cot_vision_data_for_df, columns=column_names)
@@ -146,6 +152,7 @@ def finalize_df(df):
 
 # Finalize all DataFrames
 accuracy_df = finalize_df(accuracy_df)
+accuracy_df_qwen = finalize_df(accuracy_df_qwen)
 vision_accuracy_df = finalize_df(vision_accuracy_df)
 cot_text_accuracy_df = finalize_df(cot_text_accuracy_df)
 # cot_vision_accuracy_df = finalize_df(cot_vision_accuracy_df)
@@ -156,6 +163,11 @@ def load_heatmap(evt: gr.SelectData):
     return heatmap_image
 
 
+def load_heatmap_qwen(evt: gr.SelectData):
+    heatmap_image = gr.Image(f"results_qwen/{evt.value}.jpg")
+    return heatmap_image
+
+
 def load_vision_heatmap(evt: gr.SelectData):
     heatmap_image = gr.Image(f"results-vision/{evt.value}.jpg")
     return heatmap_image
@@ -430,6 +442,13 @@ with gr.Blocks() as demo:
         heatmap_image = gr.Image(label="", show_label=False)
         leader_board.select(fn=load_heatmap, outputs=[heatmap_image])
 
+    with gr.Tab("Text-only Benchmark (Judged by Qwen)"):
+        gr.Markdown("# Text-only Leaderboard (Judged by Qwen)")
+        leader_board = gr.Dataframe(accuracy_df_qwen, headers=headers_with_icons)
+        gr.Markdown("## Heatmap")
+        heatmap_image_qwen = gr.Image(label="", show_label=False)
+        leader_board.select(fn=load_heatmap_qwen, outputs=[heatmap_image_qwen])
+
     with gr.Tab("Vision Benchmark"):
         gr.Markdown("# Vision Benchmark Leaderboard")
         leader_board_vision = gr.Dataframe(

results_qwen/CodeLlama-70b-Instruct-hf.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8eb9e387ef5e3ec72c0cf7f9fb69721135481977c3b2277cee5511312b8975cd
+size 14962036

results_qwen/CodeLlama-70b-Instruct-hf.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:203ab4a96fdca08c9155d6cec16d72eac15e35506e4ef551fa0fe8cf867c96a5
+size 14974843

results_qwen/Mixtral-8x7B-Instruct-v0.1.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1b4ad69d274c473073093be7fb46a2d1070a3db35cfdfbb196a927efc2df204f
+size 15659205

results_qwen/Mixtral-8x7B-Instruct-v0.1.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:579a96d609f2bd4650a078944535c7bf2d348f9304bf627248c1a40910815452
+size 15660660

results_qwen/claude-3-haiku-20240307.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eb5ca14f6f17cd4c3422e071a6e03dfc5504cb3709f5422a4b44c01daa00f57f
+size 17778799

results_qwen/claude-3-haiku-20240307.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e4a38e2e7e8f22ec5c7c55561a25cd51cfe1e261f4c171e9995993b9d41bb028
+size 17780948

results_qwen/claude-3-opus-20240229.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:413814671b1a00fe6b8fa308ab8fc4e76a046799d2e275c5fe51a1606d0a5062
+size 18235109

results_qwen/claude-3-opus-20240229.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be62c58b6b5e21ae49bdf5ba96ad4628d959b9843a2eba691bddd9c5bd717f1d
+size 18253951

results_qwen/deepseek-llm-67b-chat.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:668c30672d315df2c899df7627039723fc35bba099ae410792729a087a65b9fb
+size 12916783

results_qwen/deepseek-llm-67b-chat.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8e26477861326f0784a70903beadad6a4ca7dc61e18e3a76db9cc713d196dab0
+size 12957786

results_qwen/gemma-7b-it.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:abcf0738cde1e241535a90aae98338d1b9505a844c9561877e245b604676e489
+size 11246490

results_qwen/gemma-7b-it.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2cd2216230dadc2adf5a73a1b64c799f27913cf4033e7333290962c8926b9557
+size 11301306

results_qwen/gpt-3.5-turbo-0125.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3da618470a9256a02824d308cab84e80400ded429b9157c20b89ba720f708bb1
+size 8041854

results_qwen/gpt-3.5-turbo-0125.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e224821031c0d87b38a113efff8a9100fe770d067d2d94662c81ad6207718963
+size 8069783

results_qwen/gpt-4-0125-preview.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bd1c1982308ad0b1f510001f407bf50a9378dc94391993f4116c02beaa55c27f
+size 15996843

results_qwen/gpt-4-0125-preview.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2d31b186ce750657cbddcabbf6998069ecf2f6631954a8718c88313564063b30
+size 15985596

results_qwen/gpt-4-turbo-2024-04-09.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d50952850b16d84f951949aa629db60375aa2100cf86f1b9a143cdfd83fc1c30
+size 16522373

results_qwen/gpt-4-turbo-2024-04-09.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:661211c67ac6a0e6168e79df8d616b4abbc7e5d0fda335736cbe22912fade1f4
+size 16542716