add final score of vbench2

app.py

@@ -43,7 +43,6 @@ def add_new_eval(
         return "Error! Empty file!"
     if  model_link == '' or model_name_textbox == '' or contact_email == '':
         return gr.update(visible=True), gr.update(visible=False), gr.update(visible=True)
-    # upload_data=json.loads(input_file)
     upload_content = input_file
     submission_repo = Repository(local_dir=SUBMISSION_NAME, clone_from=SUBMISSION_URL, use_auth_token=HF_TOKEN, repo_type="dataset")
     submission_repo.git_pull()
@@ -53,7 +52,6 @@ def add_new_eval(
     update_time = now.strftime("%Y-%m-%d")  # Capture update time
     with open(f'{SUBMISSION_NAME}/{filename}.zip','wb') as f:
         f.write(input_file)
-    # shutil.copyfile(CSV_DIR, os.path.join(SUBMISSION_NAME, f"{input_file}"))
 
     csv_data = pd.read_csv(CSV_DIR)
 
@@ -172,7 +170,6 @@ def add_new_eval_i2v(
     update_time = now.strftime("%Y-%m-%d")  # Capture update time
     with open(f'{SUBMISSION_NAME}/{filename}.zip','wb') as f:
         f.write(input_file)
-    # shutil.copyfile(CSV_DIR, os.path.join(SUBMISSION_NAME, f"{input_file}"))
 
     csv_data = pd.read_csv(I2V_DIR)
 
@@ -260,8 +257,6 @@ def add_new_eval_i2v(
     return gr.update(visible=False), gr.update(visible=True), gr.update(visible=False)
 
 def get_normalized_df(df):
-    # final_score = df.drop('name', axis=1).sum(axis=1)
-    # df.insert(1, 'Overall Score', final_score)
     normalize_df = df.copy().fillna(0.0)
     for column in normalize_df.columns[1:-5]:
         min_val = NORMALIZE_DIC[column]['Min']
@@ -279,7 +274,6 @@ def get_normalized_i2v_df(df):
 
 
 def calculate_selected_score(df, selected_columns):
-    # selected_score = df[selected_columns].sum(axis=1)
     selected_QUALITY = [i for i in selected_columns if i in QUALITY_LIST]
     selected_SEMANTIC = [i for i in selected_columns if i in SEMANTIC_LIST]
     selected_quality_score = df[selected_QUALITY].sum(axis=1)/sum([DIM_WEIGHT[i] for i in selected_QUALITY])
@@ -291,12 +285,10 @@ def calculate_selected_score(df, selected_columns):
         return selected_semantic_score
     if selected_semantic_score.isna().any().any():
         return selected_quality_score
-    # print(selected_semantic_score,selected_quality_score )
     selected_score =  (selected_quality_score * QUALITY_WEIGHT + selected_semantic_score * SEMANTIC_WEIGHT) / (QUALITY_WEIGHT + SEMANTIC_WEIGHT)
     return selected_score.fillna(0.0)
 
 def calculate_selected_score_i2v(df, selected_columns):
-    # selected_score = df[selected_columns].sum(axis=1)
     selected_QUALITY = [i for i in selected_columns if i in I2V_QUALITY_LIST]
     selected_I2V = [i for i in selected_columns if i in I2V_LIST]
     selected_quality_score = df[selected_QUALITY].sum(axis=1)/sum([DIM_WEIGHT_I2V[i] for i in selected_QUALITY])
@@ -308,7 +300,6 @@ def calculate_selected_score_i2v(df, selected_columns):
         return selected_i2v_score
     if selected_i2v_score.isna().any().any():
         return selected_quality_score
-    # print(selected_i2v_score,selected_quality_score )
     selected_score =  (selected_quality_score * I2V_QUALITY_WEIGHT + selected_i2v_score * I2V_WEIGHT) / (I2V_QUALITY_WEIGHT + I2V_WEIGHT)
     return selected_score.fillna(0.0)
 
@@ -371,14 +362,10 @@ def get_final_score_i2v(df, selected_columns):
         df['Selected Score'] = selected_score
     else:
         df.insert(1, 'Selected Score', selected_score)
-    # df.loc[df[9:].isnull().any(axis=1), ['Total Score', 'I2V Score']] = 'N.A.'
     mask = df.iloc[:, 5:-5].isnull().any(axis=1)
     df.loc[mask, ['Total Score', 'I2V Score','Selected Score' ]] = np.nan
-    # df.fillna('N.A.', inplace=True)
     return df
 
-
-
 def get_final_score_quality(df, selected_columns):
     normalize_df = get_normalized_df(df)
     for name in normalize_df.drop('Model Name (clickable)', axis=1):
@@ -389,7 +376,6 @@ def get_final_score_quality(df, selected_columns):
         df['Quality Score'] = quality_score
     else:
         df.insert(1, 'Quality Score', quality_score)
-    # selected_score = normalize_df[selected_columns].sum(axis=1) / len(selected_columns)
     selected_score = normalize_df[selected_columns].sum(axis=1)/sum([DIM_WEIGHT[i] for i in selected_columns])
     if 'Selected Score' in df:
         df['Selected Score'] = selected_score
@@ -397,7 +383,28 @@ def get_final_score_quality(df, selected_columns):
         df.insert(1, 'Selected Score', selected_score)
     return df
 
-
+def get_final_score2(df, selected_columns):
+    category_to_dimension = {}
+
+    for key, value in VBENCH2_DIM2CAT.items():
+        if value not in category_to_dimension:
+            category_to_dimension[value] = []
+        category_to_dimension[value].append(key)
+    score_names = []
+    for cur_score in category_to_dimension:
+        score_name = f"{cur_score} Score"
+        score_names.append(score_name)
+        score = df[category_to_dimension[cur_score]].mean(axis=1)
+        if score_name in df:
+            df[score_name] = score
+        else:
+            df.insert(1, score_name, score)
+    avg_score = df[score_names].mean(axis=1)
+    if 'Total Score' in df:
+        df['Total Score'] = avg_score
+    else:
+        df.insert(1, 'Total Score', avg_score)
+    return df
 
 def get_baseline_df():
     submission_repo = Repository(local_dir=SUBMISSION_NAME, clone_from=SUBMISSION_URL, use_auth_token=HF_TOKEN, repo_type="dataset")
@@ -452,12 +459,9 @@ def get_baseline_df_2():
     submission_repo = Repository(local_dir=SUBMISSION_NAME, clone_from=SUBMISSION_URL, use_auth_token=HF_TOKEN, repo_type="dataset")
     submission_repo.git_pull()
     df = pd.read_csv(VBENCH2_DIR)
-    # df = get_final_score(df, checkbox_group.value)
-    # df = df.sort_values(by="Selected Score", ascending=False)
-    # present_columns = MODEL_INFO + checkbox_group.value
-    # print(present_columns)
+    df = get_final_score2(df, TASK_INFO_2)
+    df = df.sort_values(by="Total Score", ascending=False)
     df = df[COLUMN_NAMES_2]
-    # Add this line to display the results evaluated by VBench by default
     df = convert_scores_to_percentage(df)
     return df
 
@@ -497,27 +501,22 @@ def get_all_df2(dir=VBENCH2_DIR):
     submission_repo = Repository(local_dir=SUBMISSION_NAME, clone_from=SUBMISSION_URL, use_auth_token=HF_TOKEN, repo_type="dataset")
     submission_repo.git_pull()
     df = pd.read_csv(dir)
-    # df = get_final_score(df, selected_columns)
-    # df = df.sort_values(by="Selected Score", ascending=False)
+    df = get_final_score2(df, selected_columns)
+    df = df.sort_values(by="Selected Score", ascending=False)
     return df
     
 
 def convert_scores_to_percentage(df):
-    # Operate on every column in the DataFrame (except the'name 'column)
     if "Sampled by" in df.columns:
         skip_col =3
     else:
         skip_col =1
     print(df)
     for column in df.columns[skip_col:]:  # 假设第一列是'name'
-        # if df[column].isdigit():
-        # print(df[column])
-        # is_numeric = pd.to_numeric(df[column], errors='coerce').notna().all()
         valid_numeric_count = pd.to_numeric(df[column], errors='coerce').notna().sum()
         if valid_numeric_count > 0:
             df[column] = round(df[column] * 100,2)
             df[column] = df[column].apply(lambda x: f"{x:05.2f}%" if pd.notna(pd.to_numeric(x, errors='coerce')) else x)
-            # df[column] = df[column].apply(lambda x:  f"{x:05.2f}") + '%'
     return df
 
 def choose_all_quailty():
@@ -562,8 +561,6 @@ def on_filter_model_size_method_change(selected_columns, vbench_team_sample, vbe
 
 def on_filter_model_size_method_change_quality(selected_columns):
     updated_data = get_all_df_quality(selected_columns, QUALITY_DIR)
-    #print(updated_data)
-    # columns:
     selected_columns = [item for item in QUALITY_TAB if item in selected_columns]
     present_columns = MODEL_INFO_TAB_QUALITY + selected_columns
     updated_data = updated_data[present_columns]
@@ -571,7 +568,6 @@ def on_filter_model_size_method_change_quality(selected_columns):
     updated_data = convert_scores_to_percentage(updated_data)
     updated_headers = present_columns
     update_datatype = [DATA_TITILE_TYPE[COLUMN_NAMES.index(x)] for x in updated_headers]
-    # print(updated_data,present_columns,update_datatype)
     filter_component = gr.components.Dataframe(
         value=updated_data, 
         headers=updated_headers,
@@ -586,8 +582,6 @@ def on_filter_model_size_method_change_i2v(selected_columns,vbench_team_sample,
     updated_data = get_all_df_i2v(selected_columns, I2V_DIR)
     if vbench_team_sample:
         updated_data = updated_data[updated_data["Sampled by"] == 'VBench Team']
-    # if vbench_team_eval:
-    #     updated_data = updated_data[updated_data['Eval'] == 'VBench Team']
     selected_columns = [item for item in I2V_TAB if item in selected_columns]
     present_columns = MODEL_INFO_TAB_I2V + selected_columns
     updated_data = updated_data[present_columns]
@@ -595,7 +589,6 @@ def on_filter_model_size_method_change_i2v(selected_columns,vbench_team_sample,
     updated_data = convert_scores_to_percentage(updated_data)
     updated_headers = present_columns
     update_datatype = [DATA_TITILE_TYPE[COLUMN_NAMES_I2V.index(x)] for x in updated_headers]
-    # print(updated_data,present_columns,update_datatype)
     filter_component = gr.components.Dataframe(
         value=updated_data,
         headers=updated_headers,
@@ -631,22 +624,14 @@ def on_filter_model_size_method_change_long(selected_columns, vbench_team_sample
 
 
 def on_filter_model_size_method_change_2(vbench_team_sample, vbench_team_eval=False):
-    updated_data = get_all_df(VBENCH2_DIR)
+    updated_data = get_all_df2(VBENCH2_DIR)
     if vbench_team_sample:
         updated_data = updated_data[updated_data["Sampled by"] == 'VBench Team'] 
     if vbench_team_eval:
         updated_data = updated_data[updated_data['Evaluated by'] == 'VBench Team']
-    #print(updated_data)
-    # columns:
-    # selected_columns = [item for item in TASK_INFO if item in selected_columns]
-    # present_columns = MODEL_INFO + selected_columns
-    # updated_data = updated_data[present_columns]
-    # updated_data = updated_data.sort_values(by="Selected Score", ascending=False)
-    # updated_data = convert_scores_to_percentage(updated_data)
-    updated_headers = COLUMN_NAMES_2 
-    # print(COLUMN_NAMES,updated_headers,DATA_TITILE_TYPE )
+   
+    updated_headers = COLUMN_NAMES_2
     update_datatype = VBENCH2_TITLE_TYPE 
-    # print(updated_data,present_columns,update_datatype)
     filter_component = gr.components.Dataframe(
         value=updated_data, 
         headers=updated_headers,
@@ -759,8 +744,8 @@ with block:
                 visible=True,
                 height=700,
                 )
-            vbench_team_filter.change(fn=on_filter_model_size_method_change_2, inputs=[vbench_team_filter_2, vbench_validate_filter], outputs=data_component_2)
-            vbench_validate_filter.change(fn=on_filter_model_size_method_change_2, inputs=[vbench_team_filter_2, vbench_validate_filter], outputs=data_component_2)
+            vbench_team_filter_2.change(fn=on_filter_model_size_method_change_2, inputs=[vbench_team_filter_2, vbench_validate_filter], outputs=data_component_2)
+            vbench_validate_filter_2.change(fn=on_filter_model_size_method_change_2, inputs=[vbench_team_filter_2, vbench_validate_filter], outputs=data_component_2)
         
         with gr.TabItem("Video Quality", elem_id="vbench-tab-table", id=3):
             with gr.Accordion("INSTRUCTION", open=False):

constants.py

@@ -55,8 +55,15 @@ MODEL_INFO_2 = [
     "Sampled by",
     "Evaluated by",
     "Accessibility",
-    "Date"
+    "Date",
+    "Total Score",
+    'Creativity Score', 
+    'Commonsense Score', 
+    'Controllability Score', 
+    'Human Fidelity Score', 
+    'Physics Score'
 ]
+
 TASK_INFO_2 = [
     "Human Anatomy",
     "Human Clothes",
@@ -65,7 +72,7 @@ TASK_INFO_2 = [
     "Diversity",
     "Mechanics",
     "Material",
-    "Themotics",
+    "Thermotics",
     "Multi-View Consistency",
     "Dynamic Spatial Relationship",
     "Dynamic Attribute",    
@@ -194,7 +201,7 @@ I2V_QUALITY_WEIGHT = 1.0
 
 DATA_TITILE_TYPE = ['markdown', 'markdown', 'markdown', 'markdown', 'markdown', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number']
 I2V_TITILE_TYPE =  ['markdown', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number']
-VBENCH2_TITLE_TYPE = ['markdown', 'markdown', 'markdown', 'markdown', 'markdown', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number']
+VBENCH2_TITLE_TYPE = ['markdown', 'markdown', 'markdown', 'markdown', 'markdown', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number']
 
 SUBMISSION_NAME = "vbench_leaderboard_submission"
 SUBMISSION_URL = os.path.join("https://huggingface.co/datasets/Vchitect/", SUBMISSION_NAME)
@@ -301,4 +308,25 @@ NORMALIZE_DIC_I2V = {
     "Aesthetic Quality":{"Min": 0.0, "Max": 1.0},
     "Imaging Quality":{"Min": 0.0, "Max": 1.0},
     "Temporal Flickering":{"Min":0.6293, "Max": 1.0}
+}
+
+VBENCH2_DIM2CAT = {
+    "Human Anatomy": "Human Fidelity",
+    "Human Identity": "Human Fidelity",
+    "Human Clothes": "Human Fidelity",
+    "Diversity": "Creativity",
+    "Composition": "Creativity",
+    "Dynamic Spatial Relationship": "Controllability",
+    "Dynamic Attribute": "Controllability",
+    "Motion Order Understanding": "Controllability",
+    "Human Interaction": "Controllability",
+    "Complex Landscape": "Controllability",
+    "Complex Plot": "Controllability",
+    "Camera Motion": "Controllability",
+    "Motion Rationality": "Commonsense",
+    "Instance Preservation": "Commonsense",
+    "Mechanics": "Physics",
+    "Thermotics": "Physics",
+    "Material": "Physics",
+    "Multi-View Consistency": "Physics"
 }