chore: clean up requests-related codes

src/leaderboard/read_evals.py

@@ -91,21 +91,6 @@ class FullEvalResult:
                 results[eval_result.eval_name][get_safe_name(benchmark_name)] = value
         return [v for v in results.values()]
 
-    def update_with_request_file(self, request_path):
-        """
-        Update the request file
-        """
-        request_file = get_request_file_for_model(
-            request_path, self.retrieval_model, self.reranking_model
-        )
-
-        try:
-            with open(request_file, "r") as f:
-                request = json.load(f)
-            self.date = request.get("submitted_time", "")
-        except Exception:
-            print(f"Failed to find request file for {self.retrieval_model}, {self.reranking_model}: {request_path}")
-
 
 def get_request_file_for_model(requests_path, retrieval_model_name, reranking_model_name):
     """
@@ -130,7 +115,7 @@ def get_request_file_for_model(requests_path, retrieval_model_name, reranking_mo
     return request_file
 
 
-def get_raw_eval_results(results_path: str, requests_path: str) -> List[FullEvalResult]:
+def get_raw_eval_results(results_path: str) -> List[FullEvalResult]:
     """
     Load the evaluation results from a json file
     """
@@ -151,14 +136,7 @@ def get_raw_eval_results(results_path: str, requests_path: str) -> List[FullEval
     for model_result_filepath in model_result_filepaths:
         # create evaluation results
         eval_result = FullEvalResult.init_from_json_file(model_result_filepath)
-        # get the latest result that is finished
-        eval_result.update_with_request_file(requests_path)
-        latest_date_str = eval_result.date.replace(":", "-")
-        model_result_date_str = model_result_filepath.split('/')[-1
-        ].removeprefix("results_").removesuffix(".json")
-        if latest_date_str != model_result_date_str:
-            print(f'file skipped: {model_result_filepath}')
-            continue
+        model_result_date_str = model_result_filepath.split('/')[-1].removeprefix("results_").removesuffix(".json")
         print(f'file loaded: {model_result_filepath}')
         eval_name = eval_result.eval_name
         eval_results[eval_name] = eval_result

tests/src/leaderboard/test_read_evals.py

@@ -1,6 +1,6 @@
 from pathlib import Path
 
-from src.leaderboard.read_evals import FullEvalResult, get_raw_eval_results, get_request_file_for_model
+from src.leaderboard.read_evals import FullEvalResult, get_raw_eval_results
 
 cur_fp = Path(__file__)
 
@@ -19,26 +19,16 @@ def test_to_dict():
     result_dict = result_list[0]
     assert result_dict["Retrieval Model"] == "bge-m3"
     assert result_dict["Reranking Model"] == "bge-reranker-v2-m3"
-    assert result_dict["qa_wiki_en"] is not None
-    assert result_dict["qa_wiki_zh"] is not None
-
-
-def test_get_request_file_for_model():
-    requests_path = cur_fp.parents[2] / "toydata" / "test_requests"
-    request_file = get_request_file_for_model(requests_path, "bge-m3", "bge-reranker-v2-m3")
-    # only load the latest finished results
-    assert Path(request_file).name.removeprefix("eval_request_").removesuffix(".json") == "2023-11-21T18-10-08"
+    assert result_dict["wiki_en"] is not None
+    assert result_dict["wiki_zh"] is not None
 
 
 def test_get_raw_eval_results():
-    requests_path = cur_fp.parents[2] / "toydata" / "test_requests"
     results_path = cur_fp.parents[2] / "toydata" / "test_results" / "bge-m3"
-    results = get_raw_eval_results(results_path, requests_path)
+    results = get_raw_eval_results(results_path)
     # only load the latest results
     assert len(results) == 2
-    assert results[0].date == "2023-12-21T18:10:08"
     assert results[0].eval_name == "bge-m3_NoReranker"
-    assert len(results[0].results) == 3
+    assert len(results[0].results) == 6
     assert results[1].eval_name == "bge-m3_bge-reranker-v2-m3"
-    assert results[1].date == "2023-11-21T18:10:08"
     assert len(results[1].results) == 6

tests/toydata/test_requests/bge-m3/NoReranker/eval_request_2023-11-21T18-10-08.json

@@ -1,6 +0,0 @@
-{
-  "retrieval_model": "BAAI/bge-m3",
-  "reranking_model": "NoReranker",
-  "status": "FINISHED",
-  "submitted_time": "2023-11-21T18:10:08"
-}

tests/toydata/test_requests/bge-m3/NoReranker/eval_request_2023-12-21T18-10-08.json

@@ -1,6 +0,0 @@
-{
-  "retrieval_model": "BAAI/bge-m3",
-  "reranking_model": "NoReranker",
-  "status": "FINISHED",
-  "submitted_time": "2023-12-21T18:10:08"
-}

tests/toydata/test_requests/bge-m3/bge-reranker-v2-m3/eval_request_2023-11-21T18-10-08.json

@@ -1,6 +0,0 @@
-{
-  "retrieval_model": "BAAI/bge-m3",
-  "reranking_model": "BAAI/bge-reranker-v2-m3",
-  "status": "FINISHED",
-  "submitted_time": "2023-11-21T18:10:08"
-}

tests/toydata/test_requests/bge-m3/bge-reranker-v2-m3/eval_request_2023-12-21T18-10-08.json

@@ -1,6 +0,0 @@
-{
-  "retrieval_model": "BAAI/bge-m3",
-  "reranking_model": "BAAI/bge-reranker-v2-m3",
-  "status": "RUNNING",
-  "submitted_time": "2023-12-21T18:10:08"
-}

tests/toydata/test_results/bge-m3/NoReranker/results_2023-12-21T18-10-08.json

@@ -1,50 +0,0 @@
-[
-  {
-    "config": {
-      "retrieval_model": "bge-m3",
-      "reranking_model": "NoReranker",
-      "task": "long_doc",
-      "metric": "ndcg_at_1"
-    },
-    "results": [
-      {
-        "domain": "law",
-        "lang": "en",
-        "dataset": "lex_files_500K-600K",
-        "value": 0.45723
-      }
-    ]
-  },
-  {
-    "config": {
-      "retrieval_model": "bge-m3",
-      "reranking_model": "NoReranker",
-      "task": "qa",
-      "metric": "ndcg_at_1"
-    },
-    "results": [
-      {
-        "domain": "wiki",
-        "lang": "en",
-        "dataset": "unknown",
-        "value": 0.39083
-      }
-    ]
-  },
-  {
-    "config": {
-      "retrieval_model": "bge-m3",
-      "reranking_model": "NoReranker",
-      "task": "qa",
-      "metric": "ndcg_at_1"
-    },
-    "results": [
-      {
-        "domain": "wiki",
-        "lang": "zh",
-        "dataset": "unknown",
-        "value": 0.78358
-      }
-    ]
-  }
-]