Update app.py

app.py

@@ -1,9 +1,14 @@
+# app.py
+
 import gradio as gr
 from gradio_leaderboard import Leaderboard, ColumnFilter, SelectColumns
 import pandas as pd
 from apscheduler.schedulers.background import BackgroundScheduler
 from huggingface_hub import snapshot_download
+import os
+import json # 导入 json 和 os 库，用于处理文件
 
+# 从现有的 src 导入，这些我们无法修改，但需要继续使用其提供的功能
 from src.about import (
     CITATION_BUTTON_LABEL,
     CITATION_BUTTON_TEXT,
@@ -13,42 +18,275 @@ from src.about import (
     TITLE,
 )
 from src.display.css_html_js import custom_css
-from src.display.utils import (
-    BENCHMARK_COLS,
-    COLS,
-    EVAL_COLS,
-    EVAL_TYPES,
-    AutoEvalColumn,
-    ModelType,
-    fields,
-    WeightType,
-    Precision
-)
-from src.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, QUEUE_REPO, REPO_ID, RESULTS_REPO, TOKEN
-from src.populate import get_evaluation_queue_df, get_leaderboard_df
-from src.submission.submit import add_new_eval
 
+# =====================================================================
+# **重要修改开始：直接在 app.py 中定义 GRACE 相关的类和函数**
+#   我们无法修改 src/display/utils.py 和 src/populate.py
+#   所以在这里重新定义或覆盖部分功能，以添加 GRACE 维度。
+# =====================================================================
+
+from enum import Enum
+from typing import NamedTuple, List
+
+# 重新定义 Column 类（如果 src/display/utils 中有，这里的定义将优先被 app.py 使用）
+class Column(NamedTuple):
+    name: str
+    type: str
+    displayed_by_default: bool = True
+    never_hidden: bool = False
+    hidden: bool = False
+    filterable: bool = True
+
+# 重新定义 AutoEvalColumn，加入 GRACE 维度
+class AutoEvalColumn(Enum):
+    # 尽可能复制 src/display/utils.py 中已有的 AutoEvalColumn 定义
+    # 但请注意，如果您不知道原始的精确定义，这可能会导致不一致。
+    # 这里我将使用一个合理的通用版本，并加入 GRACE 维度。
+    # 您需要确保这些列名与您评估结果数据中的列名匹配。
+    model = Column("Model", "str", displayed_by_default=True, never_hidden=True)
+    model_type = Column("Model type", "str", displayed_by_default=True)
+    precision = Column("Precision", "str", displayed_by_default=False)
+    params = Column("Params (B)", "number", displayed_by_default=True)
+    license = Column("License", "str", displayed_by_default=False)
+    still_on_hub = Column("On Hub", "boolean", displayed_by_default=True, hidden=True)
+    # ... 您可以尝试从已运行的 Leaderboard 检查元素，推断出其他默认列 ...
+    # 例如：
+    # dataset = Column("Dataset", "str", displayed_by_default=False)
+    # average_score = Column("Average Score", "number", displayed_by_default=True) # 假设有一个总分
+
+    # GRACE 框架新增列
+    generalization_score = Column("G: 泛化性", "number", displayed_by_default=True, filterable=True)
+    relevance_score = Column("R: 相关性", "number", displayed_by_default=True, filterable=True)
+    artistry_score = Column("A: 创新表现力", "number", displayed_by_default=True, filterable=True)
+    consistency_score = Column("C: 一致性", "number", displayed_by_default=True, filterable=True)
+    efficiency_score = Column("E: 效率性", "number", displayed_by_default=True, filterable=True)
+
+# 重新定义 fields() 函数
+def fields(cls: type) -> List[Column]:
+    return [c.value for c in cls if isinstance(c.value, Column)]
+
+# 重新定义 ModelType 枚举（选择一个作为焦点，例如 LanguageModeling）
+class ModelType(Enum):
+    LanguageModeling = "语言生成模型"
+    ImageGeneration = "图像生成模型"
+    AudioSynthesis = "音频模型"
+    # ... 根据您实际的 src/display/utils.py 或项目需求添加其他类型
+    Unknown = "未知" # 保持 Unknown，防止意外
+
+    def to_str(self, sep: str = " : ") -> str:
+        return f"{self.name}{sep}{self.value}"
+
+# 重新定义 WeightType 和 Precision 枚举
+class WeightType(Enum):
+    Original = NamedTuple("Original", [("name", str)])("Original")
+    Lora = NamedTuple("Lora", [("name", str)])("Lora")
+    # Add other types if necessary from your original src/display/utils.py
+    # Example:
+    # Adapter = NamedTuple("Adapter", [("name", str)])("Adapter")
+
+class Precision(Enum):
+    float16 = NamedTuple("float16", [("name", str)])("float16")
+    bfloat16 = NamedTuple("bfloat16", [("name", str)])("bfloat16")
+    # Add other types if necessary
+    Unknown = NamedTuple("Unknown", [("name", str)])("Unknown")
+
+
+# 重新定义 COLS, BENCHMARK_COLS, EVAL_COLS, EVAL_TYPES
+# 这些列表现在将使用我们在 app.py 中定义的 AutoEvalColumn
+COLS = fields(AutoEvalColumn) # 所有列，包括 GRACE
+BENCHMARK_COLS = [
+    AutoEvalColumn.model.value,
+    AutoEvalColumn.params.value,
+    AutoEvalColumn.generalization_score.value,
+    AutoEvalColumn.relevance_score.value,
+    AutoEvalColumn.artistry_score.value,
+    AutoEvalColumn.consistency_score.value,
+    AutoEvalColumn.efficiency_score.value,
+    # ... 其他你想在基准测试中默认显示的列
+]
+EVAL_COLS = [c.name for c in fields(AutoEvalColumn)] # 评估队列的列名
+EVAL_TYPES = [c.type for c in fields(AutoEvalColumn)] # 评估队列的列类型
+
+
+# 重新定义 get_leaderboard_df 和 get_evaluation_queue_df 函数
+# 这两个函数现在将直接在 app.py 中处理数据加载和 GRACE 维度的添加。
+# 由于您无法修改 src/populate.py，我们需要在这里实现其功能。
+
+def get_leaderboard_df(eval_results_path: str, eval_requests_path: str, cols: list, benchmark_cols: list) -> pd.DataFrame:
+    """
+    加载评估结果并构建排行榜 DataFrame。
+    此函数现在在 app.py 中定义，以包含 GRACE 分数。
+    """
+    all_results = []
+
+    # ============== **重点修改区域：GRACE 分数的数据来源** ==============
+    # 您需要根据您实际的评估结果文件格式来读取数据并包含 GRACE 分数。
+    # 假设您的评估结果是在 EVAL_RESULTS_PATH 目录下，每个模型的 JSON 文件。
+    # 示例路径：EVAL_RESULTS_PATH/model_name/results.json
+    if os.path.exists(eval_results_path) and os.path.isdir(eval_results_path):
+        for model_dir in os.listdir(eval_results_path):
+            model_path = os.path.join(eval_results_path, model_dir)
+            if os.path.isdir(model_path):
+                # 尝试读取 results.json 或其他命名约定
+                results_file = os.path.join(model_path, "results.json")
+                if os.path.exists(results_file):
+                    try:
+                        with open(results_file, "r", encoding="utf-8") as f:
+                            data = json.load(f)
+                            # 确保 data 字典中包含 'generalization_score', 'relevance_score' 等键
+                            # 如果您的原始结果没有这些键，您需要在外部评估过程生成它们，或在这里进行计算。
+                            # 这里假设结果文件中直接有这些字段。
+                            all_results.append(data)
+                    except json.JSONDecodeError as e:
+                        print(f"解析 {results_file} 失败: {e}")
+                    except Exception as e:
+                        print(f"读取 {results_file} 发生未知错误: {e}")
+                else:
+                    print(f"在 {model_path} 中未找到 results.json。")
+    else:
+        print(f"评估结果路径不存在或不是目录: {eval_results_path}")
+
+    # 如果没有实际结果，提供一些模拟数据以便测试和展示 GRACE 维度
+    if not all_results:
+        print("未找到评估结果，使用模拟数据填充排行榜。")
+        all_results = [
+            {
+                "model": "模拟模型_A",
+                "model_type": ModelType.LanguageModeling.to_str(),
+                "precision": Precision.float16.value.name,
+                "params": 7.0,
+                "license": "apache-2.0",
+                "still_on_hub": True,
+                "generalization_score": 0.85,
+                "relevance_score": 0.92,
+                "artistry_score": 0.78,
+                "consistency_score": 0.88,
+                "efficiency_score": 0.95,
+                # ... 其他您希望展示的列，确保与 AutoEvalColumn 定义匹配
+            },
+            {
+                "model": "模拟模型_B",
+                "model_type": ModelType.LanguageModeling.to_str(),
+                "precision": Precision.float16.value.name,
+                "params": 13.0,
+                "license": "mit",
+                "still_on_hub": True,
+                "generalization_score": 0.90,
+                "relevance_score": 0.88,
+                "artistry_score": 0.85,
+                "consistency_score": 0.91,
+                "efficiency_score": 0.90,
+                # ...
+            },
+            {
+                "model": "模拟模型_C_图像",
+                "model_type": ModelType.ImageGeneration.to_str(),
+                "precision": Precision.bfloat16.value.name,
+                "params": 3.0,
+                "license": "gpl-3.0",
+                "still_on_hub": True,
+                "generalization_score": 0.70,
+                "relevance_score": 0.75,
+                "artistry_score": 0.90,
+                "consistency_score": None, # 图像模型可能没有一致性得分
+                "efficiency_score": 0.80,
+                # ...
+            }
+        ]
+    # =====================================================================
+
+    if all_results:
+        df = pd.DataFrame(all_results)
+    else:
+        df = pd.DataFrame(columns=[c.name for c in fields(AutoEvalColumn)])
+
+    # 确保所有期望的列都存在，如果缺失则填充 None
+    expected_cols_names = [c.name for c in cols]
+    for col_name in expected_cols_names:
+        if col_name not in df.columns:
+            df[col_name] = None
+    
+    # 对 DataFrame 进行必要的处理，例如排序
+    if AutoEvalColumn.generalization_score.value.name in df.columns and not df[AutoEvalColumn.generalization_score.value.name].isnull().all():
+        df = df.sort_values(by=AutoEvalColumn.generalization_score.value.name, ascending=False).reset_index(drop=True)
+
+    return df
+
+def get_evaluation_queue_df(eval_requests_path: str, eval_cols: list):
+    """
+    加载评估请求队列数据。此函数现在在 app.py 中定义。
+    """
+    pending_requests = []
+    running_requests = []
+    finished_requests = []
+
+    # 示例：假设请求文件是位于 eval_requests_path 的 jsonl 文件
+    if os.path.exists(eval_requests_path) and os.path.isdir(eval_requests_path):
+        for filename in os.listdir(eval_requests_path):
+            if filename.endswith(".jsonl"): # 或者其他你存储请求的文件格式
+                filepath = os.path.join(eval_requests_path, filename)
+                try:
+                    with open(filepath, "r", encoding="utf-8") as f:
+                        for line in f:
+                            try:
+                                request_data = json.loads(line)
+                                status = request_data.get('status', 'pending') # 假设请求数据中有 'status' 字段
+                                if status == 'finished':
+                                    finished_requests.append(request_data)
+                                elif status == 'running':
+                                    running_requests.append(request_data)
+                                else: # 默认或其他状态归为 pending
+                                    pending_requests.append(request_data)
+                            except json.JSONDecodeError as e:
+                                print(f"解析 JSONL 行失败: {line.strip()}, 错误: {e}")
+                except Exception as e:
+                    print(f"读取 {filepath} 失败: {e}")
+    else:
+        print(f"评估请求路径不存在或不是目录: {eval_requests_path}")
+
+    # 将列表转换为 DataFrame，并确保列与 eval_cols 匹配
+    finished_df = pd.DataFrame(finished_requests, columns=eval_cols) if finished_requests else pd.DataFrame(columns=eval_cols)
+    running_df = pd.DataFrame(running_requests, columns=eval_cols) if running_requests else pd.DataFrame(columns=eval_cols)
+    pending_df = pd.DataFrame(pending_requests, columns=eval_cols) if pending_requests else pd.DataFrame(columns=eval_cols)
+
+    return finished_df, running_df, pending_df
+
+# =====================================================================
+# **重要修改结束：直接在 app.py 中定义 GRACE 相关的类和函数**
+# =====================================================================
+
+
+# 继续使用 src.envs 中的 API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, QUEUE_REPO, REPO_ID, RESULTS_REPO, TOKEN
+# 这里我们假设这些环境变量或常量是可以通过某种方式加载的，或者在 Space 设置中配置的。
+# 如果 src.envs 也是无法修改的，且您无法通过环境变量设置这些值，那可能会有问题。
+# 通常在 Hugging Face Space 中，这些值是从环境变量或 Space Secrets 中加载的。
+# 这里我不会重定义它们，假设它们是可用的。
+from src.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, QUEUE_REPO, REPO_ID, RESULTS_REPO, TOKEN
+from src.submission.submit import add_new_eval # 假设 add_new_eval 也是从 src 导入的
 
 def restart_space():
     API.restart_space(repo_id=REPO_ID)
 
 ### Space initialisation
 try:
-    print(EVAL_REQUESTS_PATH)
+    print(f"下载评估请求到: {EVAL_REQUESTS_PATH}")
     snapshot_download(
         repo_id=QUEUE_REPO, local_dir=EVAL_REQUESTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN
     )
-except Exception:
+except Exception as e:
+    print(f"下载评估请求失败: {e}")
     restart_space()
 try:
-    print(EVAL_RESULTS_PATH)
+    print(f"下载评估结果到: {EVAL_RESULTS_PATH}")
     snapshot_download(
         repo_id=RESULTS_REPO, local_dir=EVAL_RESULTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN
     )
-except Exception:
+except Exception as e:
+    print(f"下载评估结果失败: {e}")
     restart_space()
 
 
+# 现在，这些函数调用将使用我们刚刚在 app.py 中定义的版本
 LEADERBOARD_DF = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, COLS, BENCHMARK_COLS)
 
 (
@@ -59,32 +297,84 @@ LEADERBOARD_DF = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, COLS,
 
 def init_leaderboard(dataframe):
     if dataframe is None or dataframe.empty:
-        raise ValueError("Leaderboard DataFrame is empty or None.")
+        print("Leaderboard DataFrame 为空或 None，初始化空排行榜。")
+        return Leaderboard(
+            value=pd.DataFrame(columns=[c.name for c in fields(AutoEvalColumn)]), # 提供空但带列名的DataFrame
+            datatype=[c.type for c in fields(AutoEvalColumn)],
+            select_columns=SelectColumns(
+                default_selection=[c.name for c in fields(AutoEvalColumn) if c.displayed_by_default],
+                cant_deselect=[c.name for c in fields(AutoEvalColumn) if c.never_hidden],
+                label="选择要显示的列:",
+            ),
+            search_columns=[AutoEvalColumn.model.name, AutoEvalColumn.license.name],
+            hide_columns=[c.name for c in fields(AutoEvalColumn) if c.hidden],
+            filter_columns=[], # 如果是空 DataFrame，这里不添加具体的过滤器，避免错误
+            bool_checkboxgroup_label="隐藏模型",
+            interactive=False,
+        )
+
     return Leaderboard(
         value=dataframe,
         datatype=[c.type for c in fields(AutoEvalColumn)],
         select_columns=SelectColumns(
             default_selection=[c.name for c in fields(AutoEvalColumn) if c.displayed_by_default],
             cant_deselect=[c.name for c in fields(AutoEvalColumn) if c.never_hidden],
-            label="Select Columns to Display:",
+            label="选择要显示的列:",
         ),
         search_columns=[AutoEvalColumn.model.name, AutoEvalColumn.license.name],
         hide_columns=[c.name for c in fields(AutoEvalColumn) if c.hidden],
         filter_columns=[
-            ColumnFilter(AutoEvalColumn.model_type.name, type="checkboxgroup", label="Model types"),
-            ColumnFilter(AutoEvalColumn.precision.name, type="checkboxgroup", label="Precision"),
+            ColumnFilter(AutoEvalColumn.model_type.name, type="checkboxgroup", label="模型类型"),
+            ColumnFilter(AutoEvalColumn.precision.name, type="checkboxgroup", label="精度"),
             ColumnFilter(
                 AutoEvalColumn.params.name,
                 type="slider",
                 min=0.01,
                 max=150,
-                label="Select the number of parameters (B)",
+                label="选择参数数量 (B)",
+            ),
+            ColumnFilter(
+                AutoEvalColumn.still_on_hub.name, type="boolean", label="已删除/不完整", default=True
             ),
+            # 为 GRACE 分数添加筛选器 (滑块)
+            # 假设分数在 0.0 到 1.0 之间
             ColumnFilter(
-                AutoEvalColumn.still_on_hub.name, type="boolean", label="Deleted/incomplete", default=True
+                AutoEvalColumn.generalization_score.value.name,
+                type="slider",
+                min=0.0,
+                max=1.0,
+                label="G: 泛化性得分",
+            ),
+            ColumnFilter(
+                AutoEvalColumn.relevance_score.value.name,
+                type="slider",
+                min=0.0,
+                max=1.0,
+                label="R: 相关性得分",
+            ),
+            ColumnFilter(
+                AutoEvalColumn.artistry_score.value.name,
+                type="slider",
+                min=0.0,
+                max=1.0,
+                label="A: 创新表现力得分",
+            ),
+            ColumnFilter(
+                AutoEvalColumn.consistency_score.value.name,
+                type="slider",
+                min=0.0,
+                max=1.0,
+                label="C: 一致性得分",
+            ),
+            ColumnFilter(
+                AutoEvalColumn.efficiency_score.value.name,
+                type="slider",
+                min=0.0,
+                max=1.0,
+                label="E: 效率性得分",
             ),
         ],
-        bool_checkboxgroup_label="Hide models",
+        bool_checkboxgroup_label="隐藏模型",
         interactive=False,
     )
 
@@ -98,17 +388,17 @@ with demo:
         with gr.TabItem("🏅 LLM Benchmark", elem_id="llm-benchmark-tab-table", id=0):
             leaderboard = init_leaderboard(LEADERBOARD_DF)
 
-        with gr.TabItem("📝 About", elem_id="llm-benchmark-tab-table", id=2):
+        with gr.TabItem("📝 关于", elem_id="llm-benchmark-tab-table", id=2):
             gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")
 
-        with gr.TabItem("🚀 Submit here! ", elem_id="llm-benchmark-tab-table", id=3):
+        with gr.TabItem("🚀 在此提交!", elem_id="llm-benchmark-tab-table", id=3):
             with gr.Column():
                 with gr.Row():
                     gr.Markdown(EVALUATION_QUEUE_TEXT, elem_classes="markdown-text")
 
                 with gr.Column():
                     with gr.Accordion(
-                        f"✅ Finished Evaluations ({len(finished_eval_queue_df)})",
+                        f"✅ 已完成评估 ({len(finished_eval_queue_df)})",
                         open=False,
                     ):
                         with gr.Row():
@@ -117,9 +407,10 @@ with demo:
                                 headers=EVAL_COLS,
                                 datatype=EVAL_TYPES,
                                 row_count=5,
+                                label="已完成评估队列",
                             )
                     with gr.Accordion(
-                        f"🔄 Running Evaluation Queue ({len(running_eval_queue_df)})",
+                        f"🔄 正在运行的评估队列 ({len(running_eval_queue_df)})",
                         open=False,
                     ):
                         with gr.Row():
@@ -128,10 +419,11 @@ with demo:
                                 headers=EVAL_COLS,
                                 datatype=EVAL_TYPES,
                                 row_count=5,
+                                label="正在运行的评估队列",
                             )
 
                     with gr.Accordion(
-                        f"⏳ Pending Evaluation Queue ({len(pending_eval_queue_df)})",
+                        f"⏳ 待处理的评估队列 ({len(pending_eval_queue_df)})",
                         open=False,
                     ):
                         with gr.Row():
@@ -140,40 +432,42 @@ with demo:
                                 headers=EVAL_COLS,
                                 datatype=EVAL_TYPES,
                                 row_count=5,
+                                label="待处理的评估队列",
                             )
             with gr.Row():
-                gr.Markdown("# ✉️✨ Submit your model here!", elem_classes="markdown-text")
+                gr.Markdown("# ✉️✨ 在此提交您的模型!", elem_classes="markdown-text")
 
             with gr.Row():
                 with gr.Column():
-                    model_name_textbox = gr.Textbox(label="Model name")
-                    revision_name_textbox = gr.Textbox(label="Revision commit", placeholder="main")
+                    model_name_textbox = gr.Textbox(label="模型名称")
+                    revision_name_textbox = gr.Textbox(label="修订提交", placeholder="main")
+                    # 设置模型类型的默认值，以体现项目焦点（例如：语言生成模型）
                     model_type = gr.Dropdown(
                         choices=[t.to_str(" : ") for t in ModelType if t != ModelType.Unknown],
-                        label="Model type",
+                        label="模型类型",
                         multiselect=False,
-                        value=None,
+                        value=ModelType.LanguageModeling.to_str(" : "), # 示例：聚焦于语言生成模型
                         interactive=True,
                     )
 
                 with gr.Column():
                     precision = gr.Dropdown(
                         choices=[i.value.name for i in Precision if i != Precision.Unknown],
-                        label="Precision",
+                        label="精度",
                         multiselect=False,
                         value="float16",
                         interactive=True,
                     )
                     weight_type = gr.Dropdown(
                         choices=[i.value.name for i in WeightType],
-                        label="Weights type",
+                        label="权重类型",
                         multiselect=False,
                         value="Original",
                         interactive=True,
                     )
-                    base_model_name_textbox = gr.Textbox(label="Base model (for delta or adapter weights)")
+                    base_model_name_textbox = gr.Textbox(label="基础模型 (适用于 delta 或 adapter 权重)")
 
-            submit_button = gr.Button("Submit Eval")
+            submit_button = gr.Button("提交评估")
             submission_result = gr.Markdown()
             submit_button.click(
                 add_new_eval,
@@ -189,7 +483,7 @@ with demo:
             )
 
     with gr.Row():
-        with gr.Accordion("📙 Citation", open=False):
+        with gr.Accordion("📙 引用", open=False):
             citation_button = gr.Textbox(
                 value=CITATION_BUTTON_TEXT,
                 label=CITATION_BUTTON_LABEL,
@@ -199,6 +493,7 @@ with demo:
             )
 
 scheduler = BackgroundScheduler()
-scheduler.add_job(restart_space, "interval", seconds=1800)
+# 每 30 分钟重启一次 Space，确保数据刷新
+scheduler.add_job(restart_space, "interval", seconds=1800) 
 scheduler.start()
 demo.queue(default_concurrency_limit=40).launch()