Update app.py

app.py

@@ -256,7 +256,7 @@ def get_normalized_df(df):
     # final_score = df.drop('name', axis=1).sum(axis=1)
     # df.insert(1, 'Overall Score', final_score)
     normalize_df = df.copy().fillna(0.0)
-    for column in normalize_df.columns[1:-3]:
+    for column in normalize_df.columns[1:-5]:
         min_val = NORMALIZE_DIC[column]['Min']
         max_val = NORMALIZE_DIC[column]['Max']
         normalize_df[column] = (normalize_df[column] - min_val) / (max_val - min_val)
@@ -264,7 +264,7 @@ def get_normalized_df(df):
 
 def get_normalized_i2v_df(df):
     normalize_df = df.copy().fillna(0.0)
-    for column in normalize_df.columns[1:-3]:
+    for column in normalize_df.columns[1:-5]:
         min_val = NORMALIZE_DIC_I2V[column]['Min']
         max_val = NORMALIZE_DIC_I2V[column]['Max']
         normalize_df[column] = (normalize_df[column] - min_val) / (max_val - min_val)
@@ -308,8 +308,12 @@ def calculate_selected_score_i2v(df, selected_columns):
 def get_final_score(df, selected_columns):
     normalize_df = get_normalized_df(df)
     #final_score = normalize_df.drop('name', axis=1).sum(axis=1)
-    for name in normalize_df.drop('Model Name (clickable)', axis=1).drop('Source', axis=1).drop('Mail', axis=1).drop('Date',axis=1):
-        normalize_df[name] = normalize_df[name]*DIM_WEIGHT[name]
+    try:
+        for name in normalize_df.drop('Model Name (clickable)', axis=1).drop("Sampled by", axis=1).drop('Mail', axis=1).drop('Date',axis=1).drop("Evaluated by", axis=1).drop("Accessibility", axis=1):
+            normalize_df[name] = normalize_df[name]*DIM_WEIGHT[name]
+    except:
+        for name in normalize_df.drop('Model Name (clickable)', axis=1).drop("Sampled by", axis=1).drop('Mail', axis=1).drop('Date',axis=1):
+            normalize_df[name] = normalize_df[name]*DIM_WEIGHT[name]
     quality_score = normalize_df[QUALITY_LIST].sum(axis=1)/sum([DIM_WEIGHT[i] for i in QUALITY_LIST])
     semantic_score = normalize_df[SEMANTIC_LIST].sum(axis=1)/sum([DIM_WEIGHT[i] for i in SEMANTIC_LIST ])
     final_score =  (quality_score * QUALITY_WEIGHT + semantic_score * SEMANTIC_WEIGHT) / (QUALITY_WEIGHT + SEMANTIC_WEIGHT)
@@ -335,7 +339,7 @@ def get_final_score(df, selected_columns):
 def get_final_score_i2v(df, selected_columns):
     normalize_df = get_normalized_i2v_df(df)
     #final_score = normalize_df.drop('name', axis=1).sum(axis=1)
-    for name in normalize_df.drop('Model Name (clickable)', axis=1).drop('Video-Text Camera Motion', axis=1).drop('Source', axis=1).drop('Mail', axis=1).drop('Date',axis=1):
+    for name in normalize_df.drop('Model Name (clickable)', axis=1).drop('Video-Text Camera Motion', axis=1).drop("Source", axis=1).drop('Mail', axis=1).drop('Date',axis=1):
         normalize_df[name] = normalize_df[name]*DIM_WEIGHT_I2V[name]
     quality_score = normalize_df[I2V_QUALITY_LIST].sum(axis=1)/sum([DIM_WEIGHT_I2V[i] for i in I2V_QUALITY_LIST])
     i2v_score = normalize_df[I2V_LIST].sum(axis=1)/sum([DIM_WEIGHT_I2V[i] for i in I2V_LIST ])
@@ -388,9 +392,10 @@ def get_baseline_df():
     df = get_final_score(df, checkbox_group.value)
     df = df.sort_values(by="Selected Score", ascending=False)
     present_columns = MODEL_INFO + checkbox_group.value
-    print(present_columns)
+    # print(present_columns)
     df = df[present_columns]
-    df = df[df['Source'] == 'VBench Team'] 
+    # Add this line to display the results evaluated by VBench by default
+    df = df[df['Evaluated by'] == 'VBench Team']
     df = convert_scores_to_percentage(df)
     return df
 
@@ -412,7 +417,7 @@ def get_baseline_df_i2v():
     df = get_final_score_i2v(df, checkbox_group_i2v.value)
     df = df.sort_values(by="Selected Score", ascending=False)
     present_columns = MODEL_INFO_TAB_I2V + checkbox_group_i2v.value
-    # df = df[df['Source'] == 'VBench Team'] 
+    # df = df[df["Sampled by"] == 'VBench Team'] 
     df = df[present_columns]
     df = convert_scores_to_percentage(df)
     return df
@@ -424,7 +429,7 @@ def get_baseline_df_long():
     df = get_final_score(df, checkbox_group.value)
     df = df.sort_values(by="Selected Score", ascending=False)
     present_columns = MODEL_INFO + checkbox_group.value
-    # df = df[df['Source'] == 'VBench Team'] 
+    # df = df[df["Sampled by"] == 'VBench Team'] 
     df = df[present_columns]
     df = convert_scores_to_percentage(df)
     return df
@@ -463,15 +468,19 @@ def get_all_df_long(selected_columns, dir=LONG_DIR):
 
 
 def convert_scores_to_percentage(df):
-    # 对DataFrame中的每一列（除了'name'列）进行操作
-
-    if 'Source' in df.columns:
+    # Operate on every column in the DataFrame (except the'name 'column)
+    if "Sampled by" in df.columns:
         skip_col =3
     else:
         skip_col =1
+    print(df)
     for column in df.columns[skip_col:]:  # 假设第一列是'name'
-        df[column] = round(df[column] * 100,2)  # 将分数转换为百分数
-        df[column] = df[column].apply(lambda x:  f"{x:05.2f}") + '%'
+        # if df[column].isdigit():
+        # print(df[column])
+        is_numeric = pd.to_numeric(df[column], errors='coerce').notna().all()
+        if is_numeric:
+            df[column] = round(df[column] * 100,2)
+            df[column] = df[column].apply(lambda x:  f"{x:05.2f}") + '%'
     return df
 
 def choose_all_quailty():
@@ -487,10 +496,12 @@ def enable_all():
     return gr.update(value=TASK_INFO)
 
 # select function
-def on_filter_model_size_method_change(selected_columns, only_vbench_team):
+def on_filter_model_size_method_change(selected_columns, vbench_team_sample, vbench_team_eval=False):
     updated_data = get_all_df(selected_columns, CSV_DIR)
-    if only_vbench_team:
-        updated_data = updated_data[updated_data['Source'] == 'VBench Team'] 
+    if vbench_team_sample:
+        updated_data = updated_data[updated_data["Sampled by"] == 'VBench Team'] 
+    if vbench_team_eval:
+        updated_data = updated_data[updated_data['Evaluated by'] == 'VBench Team']
     #print(updated_data)
     # columns:
     selected_columns = [item for item in TASK_INFO if item in selected_columns]
@@ -499,6 +510,7 @@ def on_filter_model_size_method_change(selected_columns, only_vbench_team):
     updated_data = updated_data.sort_values(by="Selected Score", ascending=False)
     updated_data = convert_scores_to_percentage(updated_data)
     updated_headers = present_columns
+    print(COLUMN_NAMES,updated_headers,DATA_TITILE_TYPE )
     update_datatype = [DATA_TITILE_TYPE[COLUMN_NAMES.index(x)] for x in updated_headers]
     # print(updated_data,present_columns,update_datatype)
     filter_component = gr.components.Dataframe(
@@ -533,10 +545,12 @@ def on_filter_model_size_method_change_quality(selected_columns):
         )
     return filter_component#.value
 
-def on_filter_model_size_method_change_i2v(selected_columns,only_vbench_team):
+def on_filter_model_size_method_change_i2v(selected_columns,vbench_team_sample, vbench_team_eval=False):
     updated_data = get_all_df_i2v(selected_columns, I2V_DIR)
-    if only_vbench_team:
-        updated_data = updated_data[updated_data['Source'] == 'VBench Team'] 
+    if vbench_team_sample:
+        updated_data = updated_data[updated_data["Sampled by"] == 'VBench Team']
+    # if vbench_team_eval:
+    #     updated_data = updated_data[updated_data['Eval'] == 'VBench Team']
     selected_columns = [item for item in I2V_TAB if item in selected_columns]
     present_columns = MODEL_INFO_TAB_I2V + selected_columns
     updated_data = updated_data[present_columns]
@@ -555,10 +569,12 @@ def on_filter_model_size_method_change_i2v(selected_columns,only_vbench_team):
         )
     return filter_component#.value
 
-def on_filter_model_size_method_change_long(selected_columns, only_vbench_team):
+def on_filter_model_size_method_change_long(selected_columns, vbench_team_sample, vbench_team_eval=False):
     updated_data = get_all_df_long(selected_columns, LONG_DIR)
-    if only_vbench_team:
-        updated_data = updated_data[updated_data['Source'] == 'VBench Team'] 
+    if vbench_team_sample:
+        updated_data = updated_data[updated_data["Sampled by"] == 'VBench Team']
+    if vbench_team_eval:
+        updated_data = updated_data[updated_data['Evaluated by'] == 'VBench Team']
     selected_columns = [item for item in TASK_INFO if item in selected_columns]
     present_columns = MODEL_INFO + selected_columns
     updated_data = updated_data[present_columns]
@@ -607,6 +623,11 @@ with block:
 
                 with gr.Column(scale=0.8):
                     vbench_team_filter = gr.Checkbox(
+                            label="Sampled by VBench Team (Uncheck to view all submissions)",
+                            value=False,
+                            interactive=True
+                        )
+                    vbench_validate_filter = gr.Checkbox(
                         label="Evaluated by VBench Team (Uncheck to view all submissions)",
                         value=True,
                         interactive=True
@@ -629,12 +650,13 @@ with block:
                 height=700,
                 )
     
-            choosen_q.click(choose_all_quailty, inputs=None, outputs=[checkbox_group]).then(fn=on_filter_model_size_method_change, inputs=[ checkbox_group, vbench_team_filter], outputs=data_component)
-            choosen_s.click(choose_all_semantic, inputs=None, outputs=[checkbox_group]).then(fn=on_filter_model_size_method_change, inputs=[ checkbox_group, vbench_team_filter], outputs=data_component)
+            choosen_q.click(choose_all_quailty, inputs=None, outputs=[checkbox_group]).then(fn=on_filter_model_size_method_change, inputs=[ checkbox_group, vbench_team_filter,vbench_validate_filter], outputs=data_component)
+            choosen_s.click(choose_all_semantic, inputs=None, outputs=[checkbox_group]).then(fn=on_filter_model_size_method_change, inputs=[ checkbox_group, vbench_team_filter,vbench_validate_filter], outputs=data_component)
             # enable_b.click(enable_all, inputs=None, outputs=[checkbox_group]).then(fn=on_filter_model_size_method_change, inputs=[ checkbox_group, vbench_team_filter], outputs=data_component)
-            disable_b.click(disable_all, inputs=None, outputs=[checkbox_group]).then(fn=on_filter_model_size_method_change, inputs=[ checkbox_group, vbench_team_filter], outputs=data_component)
-            checkbox_group.change(fn=on_filter_model_size_method_change, inputs=[ checkbox_group, vbench_team_filter], outputs=data_component)
-            vbench_team_filter.change(fn=on_filter_model_size_method_change, inputs=[checkbox_group, vbench_team_filter], outputs=data_component)
+            disable_b.click(disable_all, inputs=None, outputs=[checkbox_group]).then(fn=on_filter_model_size_method_change, inputs=[ checkbox_group, vbench_team_filter, vbench_validate_filter], outputs=data_component)
+            checkbox_group.change(fn=on_filter_model_size_method_change, inputs=[ checkbox_group, vbench_team_filter, vbench_validate_filter], outputs=data_component)
+            vbench_team_filter.change(fn=on_filter_model_size_method_change, inputs=[checkbox_group, vbench_team_filter, vbench_validate_filter], outputs=data_component)
+            vbench_validate_filter.change(fn=on_filter_model_size_method_change, inputs=[checkbox_group, vbench_team_filter, vbench_validate_filter], outputs=data_component)
         # Table 1
         with gr.TabItem("Video Quaity", elem_id="vbench-tab-table", id=2):
             with gr.Accordion("INSTRUCTION", open=False):
@@ -678,11 +700,17 @@ with block:
             with gr.Row():
                 with gr.Column(scale=1.0):
                     # selection for column part:
-                    vbench_team_filter_i2v = gr.Checkbox(
-                        label="Evaluated by VBench Team (Uncheck to view all submissions)",
-                        value=False,
-                        interactive=True
-                    )
+                    with gr.Row():
+                        vbench_team_filter_i2v = gr.Checkbox(
+                            label="Sampled by VBench Team (Uncheck to view all submissions)",
+                            value=False,
+                            interactive=True
+                        )
+                        vbench_validate_filter_i2v = gr.Checkbox(
+                                label="Evaluated by VBench Team (Uncheck to view all submissions)",
+                                value=False,
+                                interactive=True
+                            )
                     checkbox_group_i2v = gr.CheckboxGroup(
                         choices=I2V_TAB,
                         value=I2V_TAB,
@@ -699,8 +727,9 @@ with block:
                 visible=True,
                 )
     
-            checkbox_group_i2v.change(fn=on_filter_model_size_method_change_i2v, inputs=[checkbox_group_i2v, vbench_team_filter_i2v], outputs=data_component_i2v)
-            vbench_team_filter_i2v.change(fn=on_filter_model_size_method_change_i2v, inputs=[checkbox_group_i2v, vbench_team_filter_i2v], outputs=data_component_i2v)
+            checkbox_group_i2v.change(fn=on_filter_model_size_method_change_i2v, inputs=[checkbox_group_i2v, vbench_team_filter_i2v,vbench_validate_filter_i2v], outputs=data_component_i2v)
+            vbench_team_filter_i2v.change(fn=on_filter_model_size_method_change_i2v, inputs=[checkbox_group_i2v, vbench_team_filter_i2v,vbench_validate_filter_i2v], outputs=data_component_i2v)
+            vbench_validate_filter_i2v.change(fn=on_filter_model_size_method_change_i2v, inputs=[checkbox_group_i2v, vbench_team_filter_i2v,vbench_validate_filter_i2v], outputs=data_component_i2v)
         
         with gr.TabItem("📊 VBench-Long", elem_id="vbench-tab-table", id=4):
             with gr.Row():
@@ -723,11 +752,17 @@ with block:
                     disable_b_long = gr.Button("Deselect All")
 
                 with gr.Column(scale=0.8):
-                    vbench_team_filter_long = gr.Checkbox(
-                        label="Evaluated by VBench Team (Uncheck to view all submissions)",
-                        value=False,
-                        interactive=True
-                    )
+                    with gr.Row():
+                        vbench_team_filter_long = gr.Checkbox(
+                            label="Sampled by VBench Team (Uncheck to view all submissions)",
+                            value=False,
+                            interactive=True
+                        )
+                        vbench_validate_filter_long = gr.Checkbox(
+                            label="Evaluated by VBench Team (Uncheck to view all submissions)",
+                            value=False,
+                            interactive=True
+                        )
                     checkbox_group_long = gr.CheckboxGroup(
                         choices=TASK_INFO,
                         value=DEFAULT_INFO,
@@ -745,12 +780,14 @@ with block:
                 height=700,
                 )
     
-            choosen_q_long.click(choose_all_quailty, inputs=None, outputs=[checkbox_group_long]).then(fn=on_filter_model_size_method_change_long, inputs=[ checkbox_group_long, vbench_team_filter_long], outputs=data_component)
-            choosen_s_long.click(choose_all_semantic, inputs=None, outputs=[checkbox_group_long]).then(fn=on_filter_model_size_method_change_long, inputs=[ checkbox_group_long, vbench_team_filter_long], outputs=data_component)
-            enable_b_long.click(enable_all, inputs=None, outputs=[checkbox_group_long]).then(fn=on_filter_model_size_method_change_long, inputs=[ checkbox_group_long, vbench_team_filter_long], outputs=data_component)
-            disable_b_long.click(disable_all, inputs=None, outputs=[checkbox_group_long]).then(fn=on_filter_model_size_method_change_long, inputs=[ checkbox_group_long, vbench_team_filter_long], outputs=data_component)
-            checkbox_group_long.change(fn=on_filter_model_size_method_change_long, inputs=[checkbox_group_long, vbench_team_filter_long], outputs=data_component)
-            vbench_team_filter_long.change(fn=on_filter_model_size_method_change_long, inputs=[checkbox_group_long, vbench_team_filter_long], outputs=data_component)
+            choosen_q_long.click(choose_all_quailty, inputs=None, outputs=[checkbox_group_long]).then(fn=on_filter_model_size_method_change_long, inputs=[ checkbox_group_long, vbench_team_filter_long, vbench_validate_filter_long], outputs=data_component)
+            choosen_s_long.click(choose_all_semantic, inputs=None, outputs=[checkbox_group_long]).then(fn=on_filter_model_size_method_change_long, inputs=[ checkbox_group_long, vbench_team_filter_long, vbench_validate_filter_long], outputs=data_component)
+            enable_b_long.click(enable_all, inputs=None, outputs=[checkbox_group_long]).then(fn=on_filter_model_size_method_change_long, inputs=[ checkbox_group_long, vbench_team_filter_long, vbench_validate_filter_long], outputs=data_component)
+            disable_b_long.click(disable_all, inputs=None, outputs=[checkbox_group_long]).then(fn=on_filter_model_size_method_change_long, inputs=[ checkbox_group_long, vbench_team_filter_long, vbench_validate_filter_long], outputs=data_component)
+            checkbox_group_long.change(fn=on_filter_model_size_method_change_long, inputs=[checkbox_group_long, vbench_team_filter_long,vbench_validate_filter_long], outputs=data_component)
+            vbench_team_filter_long.change(fn=on_filter_model_size_method_change_long, inputs=[checkbox_group_long, vbench_team_filter_long,vbench_validate_filter_long], outputs=data_component)
+            vbench_validate_filter_long.change(fn=on_filter_model_size_method_change_long, inputs=[checkbox_group_long, vbench_team_filter_long,vbench_validate_filter_long], outputs=data_component)
+
         # table info
         with gr.TabItem("📝 About", elem_id="mvbench-tab-table", id=5):
             gr.Markdown(LEADERBORAD_INFO, elem_classes="markdown-text")
@@ -775,6 +812,7 @@ with block:
                     revision_name_textbox = gr.Textbox(
                         label="Revision Model Name(Optional)", placeholder="If you need to update the previous results, please fill in this line"
                     )
+                    access_type = gr.Dropdown(["Open Source", "Ready to Open Source", "API", "Close"], label="Please select the way user can access your model. You can update the content by revision_name, or contact the VBench Team.")
 
                 with gr.Column():
                     model_link = gr.Textbox(
@@ -815,6 +853,7 @@ with block:
                         team_name,
                         contact_email,
                         release_time,
+                        access_type,
                         model_resolution,
                         model_fps,
                         model_frame,
@@ -885,6 +924,7 @@ with block:
                         team_name_i2v,
                         contact_email_i2v,
                         release_time_i2v,
+                        access_type,
                         model_resolution_i2v,
                         model_fps_i2v,
                         model_frame_i2v,