bugfix

app.py

@@ -21,12 +21,10 @@ from content import (
     format_log,
 )
 from evaluator import Evaluator
-from score import init_evaluators, score_item
+from score import init_evaluators, score_item, load_agent_output_dataset, polish_scores
 from loguru import logger
 
 
-# from src.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, QUEUE_REPO, REPO_ID, RESULTS_REPO, TOKEN
-
 from datasets import load_dataset, VerificationMode, Dataset, concatenate_datasets
 
 from utils import parse_eval_dataset, parseaddr
@@ -41,7 +39,8 @@ from env import (
     SUBMISSION_DATASET,
     INTERNAL_DATASET,
     EVALUATE_RESULT_DATASET,
-    REPO_ID
+    REPO_ID,
+    CONTACT_DATASET
 )
 
 TOKEN = os.getenv("HF_TOKEN")
@@ -61,6 +60,41 @@ benchmark_dataset = parse_eval_dataset(benchmark_internal_evaluate_dataset) # ty
 evaluator_list = init_evaluators(benchmark_dataset, llm_config)
 
 
+
+def save_contact_info(contact_info):
+    import tempfile
+    import json
+    
+    # 加载现有联系人信息
+    try:
+        contact_infos = load_dataset(
+            CONTACT_DATASET, 
+            data_files=CONTACT_DATASET_FILE, 
+            token=TOKEN,
+            download_mode="force_redownload", 
+            verification_mode=VerificationMode.NO_CHECKS, 
+            trust_remote_code=True
+        )
+        contact_info_list = list(contact_infos['train'])
+    except Exception as e:
+        print(f"Error loading contact info: {e}")
+        contact_info_list = []
+    
+    with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as temp_file:
+        json.dump(contact_info_list, temp_file, default=str, indent=4)
+        temp_file_path = temp_file.name
+    
+    API.upload_file(
+        path_or_fileobj=temp_file_path,
+        path_in_repo=CONTACT_DATASET_FILE,
+        repo_id=CONTACT_DATASET,
+        repo_type='dataset',
+        token=TOKEN,
+        commit_message=f"Add new contact: {contact_info['model']} by {contact_info['organisation']}"
+    )
+    
+    os.unlink(temp_file_path)
+
 def get_dataframe_from_results(eval_results, split:str = 'train'):
     try:
         if hasattr(eval_results, "__getitem__"):
@@ -80,7 +114,6 @@ def get_dataframe_from_results(eval_results, split:str = 'train'):
             print(f"Error applying model hyperlink: {e}")
             pass
             
-        # 重命名列
         column_renames = {
             "model": "Agent name",
             "model_family": "Model family",
@@ -108,7 +141,6 @@ def get_dataframe_from_results(eval_results, split:str = 'train'):
         except:
             pass
             
-        # 处理数值
         try:
             numeric_cols = [c for c in df.columns if "score" in c.lower()]
             if numeric_cols:
@@ -157,6 +189,14 @@ def add_new_eval(
     profile: gr.OAuthProfile, 
 ):
     try:
+        # 检查文件是否为JSON格式
+        try:
+            with open(path_to_file, 'r', encoding='utf-8') as f:
+                json.load(f)  # 尝试解析JSON
+        except json.JSONDecodeError:
+            return styled_error("Please upload a valid JSON file.")
+        except Exception as e:
+            return styled_error(f"File read error: {str(traceback.format_exc())}")
         if not LOCAL_DEBUG:
             print(profile)
             print(path_to_file)
@@ -168,12 +208,11 @@ def add_new_eval(
             if datetime.datetime.now() - datetime.datetime.strptime(creation_date, '%Y-%m-%dT%H:%M:%S.%fZ') < datetime.timedelta(days=1):
                 return styled_error("This account is not authorized to submit on CAIA.")
             
-            contact_infos = load_dataset(INTERNAL_DATASET, data_files=CONTACT_DATASET_FILE, token=TOKEN, 
+            contact_infos = load_dataset(CONTACT_DATASET, data_files=CONTACT_DATASET_FILE, token=TOKEN, 
                                         download_mode="force_redownload", 
                                         verification_mode=VerificationMode.NO_CHECKS, 
                                         trust_remote_code=True)
             
-            # print("Contact infos features:", contact_infos['train'].features)
             
             user_submission_dates = []
             try:
@@ -185,6 +224,7 @@ def add_new_eval(
                 print(f"Error getting user submission dates: {e}")
             
             user_submission_dates = sorted(user_submission_dates)
+            user_submission_dates = [date.strftime('%Y-%m-%d') if isinstance(date, pd.Timestamp) else datetime.datetime.strptime(str(date), '%Y-%m-%d %H:%M:%S.%f').strftime('%Y-%m-%d') for date in user_submission_dates if date]
             if len(user_submission_dates) > 0 and user_submission_dates[-1] == datetime.datetime.today().strftime('%Y-%m-%d'):
                 return styled_error("You already submitted once today, please try again tomorrow.")
 
@@ -228,31 +268,69 @@ def add_new_eval(
             "organisation": organisation,
             "username": profile.username,
             "mail": mail,
-            "date": pd.Timestamp(datetime.datetime.now()).floor('ns')
+            "date": pd.Timestamp(datetime.datetime.now())
         }
-        print("contact_info", contact_info)
-        temp_file_path = "temp_contact_info.json"
-        with open(temp_file_path, 'w') as f:
-            json.dump(contact_info_list, f)
-        to_add = Dataset.from_list([contact_info], features=contact_infos['train'].features)
-        new_data= concatenate_datasets([contact_infos['train'], to_add])
-        contact_infos['train'] = new_data
         if LOCAL_DEBUG:
             print("mock uploaded contact info")
         else:
-            contact_infos.push_to_hub(INTERNAL_DATASET, config_name = VERSION, token=TOKEN)
-            upload_file(
-                path_or_fileobj=temp_file_path,
-                path_in_repo=CONTACT_DATASET_FILE,  # 目标JSON文件路径
-                repo_id=INTERNAL_DATASET,
-                token=TOKEN
-            )
+            save_contact_info(contact_info)
 
         # SCORE SUBMISSION
         file_path = path_to_file.name        
         print("模拟评分过程...")
-        # with open(file_path, 'r') as f:
-        #     ...
+
+
+        agent_output = load_agent_output_dataset(dataset_path=file_path)
+        agent_output_task_ids = set(output.task_id for output in agent_output)
+        benchmark_task_ids = set(item.task_id for item in benchmark_dataset)
+        if agent_output_task_ids != benchmark_task_ids:
+            return styled_error("The task IDs in agent outputs do not match the task IDs in benchmark dataset.")
+        l1,l2,l3 = [],[],[]
+        for output in agent_output:
+            task_id = output.task_id
+            to_evaluate_item = [item for item in benchmark_dataset if item.task_id == task_id]
+            if not to_evaluate_item:
+                # score,detail_result = 0.0, None
+                continue
+            else:
+                level = to_evaluate_item[0].level
+                score, detail_result = asyncio.run(score_item(evaluator_list=evaluator_list, agent_output_item=output, to_evaluate_item=to_evaluate_item[0]))
+                print(score, task_id, level)
+                if level == 1:
+                    l1.append((score, detail_result))
+                elif level == 2:
+                    l2.append((score, detail_result))
+                elif level == 3:
+                    l3.append((score, detail_result))
+        l1_scores = polish_scores([item[1] for item in l1])
+        l2_scores = polish_scores([item[1] for item in l2])
+        l3_scores = polish_scores([item[1] for item in l3])
+        print(l1_scores, l2_scores, l3_scores)
+
+        l1_total_score = sum(l1_scores) / len(l1_scores)
+        l2_total_score = sum(l2_scores) / len(l2_scores)
+        l3_total_score = sum(l3_scores) / len(l3_scores)
+
+        total_score = round((sum(l1_scores) + sum(l2_scores) + sum(l3_scores)) / (len(l1) + len(l2) + len(l3)), 2)
+
+        # add to eval_results
+        new_eval_result = {
+            "model": model,
+            "model_family": model_family,
+            "url": url,
+            "organisation": organisation,
+            "score": total_score,
+            "score_level1": l1_total_score,
+            "score_level2": l2_total_score,
+            "score_level3": l3_total_score,
+            "date": datetime.datetime.now().strftime("%Y-%m-%d")
+        }
+        print(new_eval_result)
+
+        eval_results_list = list(eval_results)
+        eval_results_list.append(new_eval_result)
+        eval_results = Dataset.from_list(eval_results_list, features=eval_results.features)
+        eval_results.push_to_hub(EVALUATE_RESULT_DATASET, token=TOKEN)
         
         return format_log(f"模型 {model} 由 {organisation} 提交成功。\n请等待几个小时后刷新排行榜查看您的分数。")
     except Exception as e:
@@ -318,13 +396,6 @@ with demo:
             ],
             submission_result,
         )
-        with gr.Row():
-            new_sub_btn = gr.Button("New submission")
-            new_sub_btn.click(
-                new_submission,
-                inputs=[],
-                outputs=[],
-            )
             
 scheduler = BackgroundScheduler()
 scheduler.add_job(restart_space, "interval", seconds=3600)

env.py

@@ -49,4 +49,4 @@ CONTACT_DATASET = f"{OWNER}/contact_info"
 
 BENCHMARK_INTERNAL_EVALUATE_DATASET_FILE = f"{VERSION}/{os.getenv('BENCHMARK_INTERNAL_EVALUATE_DATASET', 'example_evaluate_data.json')}"
 EVALUATE_RESULT_DATASET_FILE = f"{VERSION}/{os.getenv('EVALUATE_RESULT_DATASET', 'example_result.json')}"
-CONTACT_DATASET_FILE = f"{VERSION}/{os.getenv('CONTACT_DATASET', 'example_contact.json')}"
+CONTACT_DATASET_FILE = f"{os.getenv('CONTACT_DATASET_FILE', 'example_contact_info.json')}"

evaluator.py

@@ -276,12 +276,8 @@ Evaluation Rules:
                 continue
         return 0.0, None
 
-    async def a_evaluate(self, task_id:str, answer:Answer) -> EvaluateScore | None:
+    async def a_evaluate(self, task_id:str, answer:Answer, to_evaluate_item: BenchmarkItem) -> EvaluateScore | None:
         import asyncio
-        to_evaluate_item = [item for item in self.benchmark_data if item.task_id == task_id]
-        if not to_evaluate_item:
-            return None
-        to_evaluate_item = to_evaluate_item[0]
         tasks = [
             self.evaluate_answer(answer, to_evaluate_item),
             self.evaluate_reasoning(answer, to_evaluate_item),
@@ -311,6 +307,7 @@ Evaluation Rules:
         detail += f"Tool use score: {sum([item.score for item in tool_use_evaulate_result.items])} / {sum([item.points for item in benchmark_tool_use_items])}\n"
         for item in tool_use_evaulate_result.items:
             detail += f"{item.reason}\n"
+        print(detail)
         return EvaluateScore(
             model_name=self.model_name,
             answer_score=answer_evaulate_result.score,
@@ -327,6 +324,6 @@ Evaluation Rules:
 async def ensemble_evaluate(evaulator_list:list[Evaluator], answer:Answer, to_evaluate_item:BenchmarkItem) -> tuple[float, list[EvaluateScore]]:
     # for evaluator in evaulator_list:
     #     await evaluator.load_validate_data()
-    results = await asyncio.gather(*[evaluator.a_evaluate(to_evaluate_item.task_id, answer) for evaluator in evaulator_list])
+    results = await asyncio.gather(*[evaluator.a_evaluate(to_evaluate_item.task_id, answer, to_evaluate_item) for evaluator in evaulator_list])
     return sum([result.total_score for result in results if result]) / len([result for result in results if result]), [result for result in results if result]
 

schemas.py

@@ -88,6 +88,7 @@ class QuestionData(BaseModel):
 
 class BenchmarkItem(BaseModel):
     task_id: str
+    level:Optional[int] = 1
     question: str = Field(description="The question to be answered")
     # answer: Answer = Field(description="The agent system output")
     evaluate: EvaluateData = Field(description="The evaluation result")

score.py

@@ -37,43 +37,6 @@ async def run_evaluate(evaluator_list:list[Evaluator], agent_output_item:AgentOu
     )
     return await ensemble_evaluate(evaluator_list, answer, to_evaluate_item)
 
-# async def main():
-#     #load llm config
-#     parse_llm_config = llm_configs["parse_llm_config"]
-#     evaluate_llm_configs = llm_configs["evaluate_llm_configs"]
-#     #load agent output dataset
-#     agent_output_dataset = load_agent_output_dataset()
-#     #load evaluate dataset
-#     evaluator_list: list[Evaluator] = []
-#     for evaluate_llm_config in evaluate_llm_configs:
-#         for _ in range(3):
-#             evaluator = Evaluator(
-#                 dataset_path="dataset/example_evaluate_data.json",
-#                 parse_model=parse_llm_config["model_name"],
-#                 parse_model_api_key=parse_llm_config.get("api_key", None),
-#                 parse_model_base_url=parse_llm_config.get("base_url", None),
-#                 api_key=evaluate_llm_config.get("api_key", None),
-#                 model_name=evaluate_llm_config["model_name"],
-#                 base_url=evaluate_llm_config.get("base_url", None),
-#                 **evaluate_llm_config.get("model_params",{})
-#             )
-#             evaluator_list.append(evaluator)
-#     evaluate_dataset = await evaluator.load_validate_data()
-#     #evaluate
-#     # run parallel
-#     for agent_output_item in agent_output_dataset:
-#         task_id = agent_output_item.task_id
-#         to_evaluate_item = [item for item in evaluate_dataset if item.task_id == task_id][0]
-#         answer = Answer(
-#             answer=agent_output_item.answer,
-#             reasoning_steps=agent_output_item.reasoning_list,
-#             function_calls=agent_output_item.tool_use_list
-#         )
-#         score,results = await ensemble_evaluate(evaluator_list, answer, to_evaluate_item)
-#         print(f"Task ID: {task_id}")
-#         print(f"Score: {score}")
-#         # print(results)
-
 async def score_item(evaluator_list:list[Evaluator], agent_output_item:AgentOutputItem, to_evaluate_item:BenchmarkItem) -> tuple[float, list[EvaluateScore]]:
     answer = Answer(
         answer=agent_output_item.answer,
@@ -82,3 +45,15 @@ async def score_item(evaluator_list:list[Evaluator], agent_output_item:AgentOutp
     )
     return await ensemble_evaluate(evaluator_list, answer, to_evaluate_item)
 
+
+
+
+def polish_scores(scores:list[EvaluateScore]) -> tuple[float, float, float]:
+    answer_scores = [score.answer_score for score in scores]
+    total_answer_scores = [score.answer_total_score for score in scores]
+    reasoning_scores = [score.reasoning_score for score in scores]
+    total_reasoning_scores = [score.reasoning_total_score for score in scores]
+    tool_use_scores = [score.tool_use_score for score in scores]
+    total_tool_use_scores = [score.tool_use_total_score for score in scores]
+    return sum(answer_scores) / sum(total_answer_scores), sum(reasoning_scores) / sum(total_reasoning_scores), sum(tool_use_scores) / sum(total_tool_use_scores)
+