update

backend-cli.py

@@ -105,7 +105,6 @@ def process_finished_requests() -> bool:
 
     for eval_request in eval_requests:
         result_name: str = request_to_result_name(eval_request)
-        print(result_name, result_name in result_name_to_result)
 
         # Check the corresponding result
         eval_result: EvalResult = result_name_to_result[result_name]
@@ -115,7 +114,6 @@ def process_finished_requests() -> bool:
             task_name = task.benchmark
 
             if task_name not in eval_result.results:
-                print(task_name)
                 results = process_evaluation(task, eval_request)
                 return True
 

beta-cli.py

@@ -4,13 +4,63 @@ from huggingface_hub import snapshot_download
 from src.leaderboard.read_evals import get_raw_eval_results
 from src.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, QUEUE_REPO, RESULTS_REPO
 
+from src.backend.run_eval_suite import run_evaluation
+from src.backend.manage_requests import check_completed_evals, get_eval_requests, set_eval_request
+from src.backend.sort_queue import sort_models_by_priority
+from src.backend.envs import Tasks, EVAL_REQUESTS_PATH_BACKEND, EVAL_RESULTS_PATH_BACKEND, DEVICE, LIMIT, Task
+
+from src.leaderboard.read_evals import get_raw_eval_results
+
+from src.backend.manage_requests import EvalRequest
+from src.leaderboard.read_evals import EvalResult
+
 snapshot_download(repo_id=QUEUE_REPO, local_dir=EVAL_REQUESTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30)
 snapshot_download(repo_id=RESULTS_REPO, local_dir=EVAL_RESULTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30)
 
-raw_data = get_raw_eval_results(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH)
+PENDING_STATUS = "PENDING"
+RUNNING_STATUS = "RUNNING"
+FINISHED_STATUS = "FINISHED"
+FAILED_STATUS = "FAILED"
+
+TASKS_HARNESS = [task.value for task in Tasks]
+
+current_finished_status = [FINISHED_STATUS]
+
+
+def request_to_result_name(request: EvalRequest) -> str:
+    org_and_model = request.model.split("/", 1)
+    if len(org_and_model) == 1:
+        model = org_and_model[0]
+        res = f"{model}_{request.precision}"
+    else:
+        org = org_and_model[0]
+        model = org_and_model[1]
+        res = f"{org}_{model}_{request.precision}"
+    return res
+
+
+# Get all eval request that are FINISHED, if you want to run other evals, change this parameter
+eval_requests: list[EvalRequest] = get_eval_requests(job_status=current_finished_status, hf_repo=QUEUE_REPO, local_dir=EVAL_REQUESTS_PATH_BACKEND)
+# Sort the evals by priority (first submitted first run)
+eval_requests: list[EvalRequest] = sort_models_by_priority(api=API, models=eval_requests)
+
+eval_results: list[EvalResult] = get_raw_eval_results(EVAL_RESULTS_PATH_BACKEND, EVAL_REQUESTS_PATH_BACKEND)
+
+result_name_to_request = {request_to_result_name(r): r for r in eval_requests}
+result_name_to_result = {r.eval_name: r for r in eval_results}
+
+print('Requests', sorted(result_name_to_request.keys()))
+print('Results', sorted(result_name_to_result.keys()))
+
+for eval_request in eval_requests:
+    result_name: str = request_to_result_name(eval_request)
+
+    # Check the corresponding result
+    eval_result: EvalResult = result_name_to_result[result_name]
 
-for entry in raw_data:
-    # if '125m' in entry.eval_name:
-    print(entry)
+    # Iterate over tasks and, if we do not have results for a task, run the relevant evaluations
+    for task in TASKS_HARNESS:
+        task_name = task.benchmark
 
-# print(raw_data)
+        if task_name not in eval_result.results:
+            print('RUN THIS ONE!', result_name, task_name)

src/leaderboard/read_evals.py

@@ -144,7 +144,8 @@ class EvalResult:
         }
 
         for task in Tasks:
-            data_dict[task.value.col_name] = self.results[task.value.benchmark]
+            if task.value.benchmark in self.results:  # XXX
+                data_dict[task.value.col_name] = self.results[task.value.benchmark]
 
         return data_dict
 
@@ -209,12 +210,6 @@ def get_raw_eval_results(results_path: str, requests_path: str) -> list[EvalResu
 
     results = []
     for v in eval_results.values():
-        try:
-            v.to_dict()  # we test if the dict version is complete
-            results.append(v)
-        except KeyError:  # not all eval values present
-            continue
-
-    # print('XXX', results_path, requests_path, results)
+        results.append(v)
 
     return results