Permit different revision

src/display/formatting.py

@@ -24,7 +24,7 @@ def make_requests_clickable_model(model_name, json_path=None):
 
     return model_hyperlink(link, model_name) + "  " + model_hyperlink(details_link, "📑")
 
-def make_clickable_model(model_name, json_path=None):
+def make_clickable_model(model_name, json_path=None, revision=None):
     link = f"https://huggingface.co/{model_name}"
 
     #details_model_name = model_name.replace("/", "__")
@@ -35,6 +35,10 @@ def make_clickable_model(model_name, json_path=None):
         if json_path is not None:
             details_link = f"https://huggingface.co/datasets/{RESULTS_REPO}/blob/main/{model_name}/{json_path}"
 
+    if revision is not None and revision != "main":
+        if len(revision) > 12:
+            revision = revision[:7]
+        model_name += f" (rev: {revision})"
     return model_hyperlink(link, model_name) + "  " + model_hyperlink(details_link, "📑")
 
 

src/display/utils.py

@@ -61,7 +61,8 @@ auto_eval_column_dict.append(["license", ColumnContent, ColumnContent("Hub Licen
 auto_eval_column_dict.append(["params", ColumnContent, ColumnContent("#Params (B)", "number", False)])
 auto_eval_column_dict.append(["likes", ColumnContent, ColumnContent("Hub ❤️", "number", False)])
 auto_eval_column_dict.append(["still_on_hub", ColumnContent, ColumnContent("Available on the hub", "bool", False, hidden=True)])
-auto_eval_column_dict.append(["revision", ColumnContent, ColumnContent("Model sha", "str", False, False)])
+auto_eval_column_dict.append(["model_sha", ColumnContent, ColumnContent("Model sha", "str", False, False)])
+auto_eval_column_dict.append(["revision", ColumnContent, ColumnContent("Revision", "str", False, False)])
 auto_eval_column_dict.append(["flagged", ColumnContent, ColumnContent("Flagged", "bool", False, hidden=True)])
 auto_eval_column_dict.append(["moe", ColumnContent, ColumnContent("MoE", "bool", False, hidden=True)])
 auto_eval_column_dict.append(["eval_time", ColumnContent, ColumnContent("Evaluation Time (s)", "number", False)])
@@ -88,6 +89,7 @@ class EvalQueueColumn:  # Queue column
 baseline_row = {
     AutoEvalColumn.model.name: "<p>Baseline</p>",
     AutoEvalColumn.revision.name: "N/A",
+    AutoEvalColumn.model_sha.name: "N/A",
     AutoEvalColumn.precision.name: "?",
     AutoEvalColumn.merged.name: False,
     #AutoEvalColumn.average.name: 31.0,
@@ -131,6 +133,7 @@ if GET_ORIGINAL_HF_LEADERBOARD_EVAL_RESULTS:
 human_baseline_row = {
     AutoEvalColumn.model.name: "<p>Human performance</p>",
     AutoEvalColumn.revision.name: "N/A",
+    AutoEvalColumn.model_sha.name: "N/A",
     AutoEvalColumn.precision.name: "?",
     #AutoEvalColumn.average.name: 92.75,
     AutoEvalColumn.merged.name: False,

src/leaderboard/read_evals.py

@@ -22,7 +22,8 @@ class EvalResult:
     full_model: str # org/model (path on hub)
     org: str 
     model: str
-    revision: str # commit hash, "" if main
+    model_sha: str # commit hash, "" if main
+    revision: str = "main"
     results: dict
     precision: Precision = Precision.Unknown
     model_type: ModelType = ModelType.Unknown # Pretrained, fine tuned, ...
@@ -120,8 +121,9 @@ class EvalResult:
             org=org,
             model=model,
             results=results,
-            precision=precision,  
-            revision= config.get("model_sha", ""),
+            precision=precision, 
+            model_sha=config.get("model_sha", ""),
+            revision=config.get("model_revision", "main"),
             json_filename=json_filename,
             eval_time=config.get("total_evaluation_time_seconds", 0.0),
             num_params=num_params
@@ -129,7 +131,7 @@ class EvalResult:
 
     def update_with_request_file(self, requests_path):
         """Finds the relevant request file for the current model and updates info with it"""
-        request_file = get_request_file_for_model(requests_path, self.full_model, self.precision.value.name)
+        request_file = get_request_file_for_model(requests_path, self.full_model, self.precision.value.name, self.revision)
 
         try:
             with open(request_file, "r") as f:
@@ -144,7 +146,7 @@ class EvalResult:
             self.main_language = request.get("main_language", "?")
         except Exception as e:
             self.status = "FAILED"
-            print(f"Could not find request file for {self.org}/{self.model}")
+            print(f"Could not find request file for {self.org}/{self.model}, precision {self.precision.value.name}")
 
     def update_with_dynamic_file_dict(self, file_dict):
         self.license = file_dict.get("license", "?")
@@ -174,6 +176,12 @@ class EvalResult:
         average = round(sum(average)/len(average), 2)
         npm = round(sum(npm)/len(npm), 2)
 
+        rev_name = None
+        if self.revision != "main":
+            rev_name = self.revision
+            if rev_name > 10:
+                rev_name = rev_name[:7]
+
         data_dict = {
             "eval_name": self.eval_name,  # not a column, just a save name,
             AutoEvalColumn.precision.name: self.precision.value.name,
@@ -181,7 +189,7 @@ class EvalResult:
             AutoEvalColumn.model_type_symbol.name: self.model_type.value.symbol,
             AutoEvalColumn.weight_type.name: self.weight_type.value.name,
             AutoEvalColumn.architecture.name: self.architecture,
-            AutoEvalColumn.model.name: make_clickable_model(self.full_model, self.json_filename),
+            AutoEvalColumn.model.name: make_clickable_model(self.full_model, self.json_filename, revision=rev_name),
             AutoEvalColumn.dummy.name: self.full_model,
             AutoEvalColumn.revision.name: self.revision,
             AutoEvalColumn.average.name: average,
@@ -207,7 +215,7 @@ class EvalResult:
         return data_dict
 
 
-def get_request_file_for_model(requests_path, model_name, precision):
+def get_request_file_for_model(requests_path, model_name, precision, revision):
     """Selects the correct request file for a given model. Only keeps runs tagged as FINISHED"""
     request_files = os.path.join(
         requests_path,
@@ -215,15 +223,21 @@ def get_request_file_for_model(requests_path, model_name, precision):
     )
     request_files = glob.glob(request_files)
 
+    if revision is None or revision == "":
+        revision = "main"
+
     # Select correct request file (precision)
     request_file = ""
     request_files = sorted(request_files, reverse=True)
     for tmp_request_file in request_files:
         with open(tmp_request_file, "r") as f:
             req_content = json.load(f)
+            if req_content["revision"] is None or req_content["revision"] == "":
+                req_content["revision"] = "main"
             if (
                 req_content["status"] in ["FINISHED", "PENDING_NEW_EVAL" if SHOW_INCOMPLETE_EVALS else "FINISHED"]
                 and req_content["precision"] == precision.split(".")[-1]
+                and req_content["revision"] == revision
             ):
                 request_file = tmp_request_file
     return request_file

src/submission/submit.py

@@ -146,7 +146,13 @@ def add_new_eval(
     print("Creating eval file")
     OUT_DIR = f"{EVAL_REQUESTS_PATH}/{user_name}"
     os.makedirs(OUT_DIR, exist_ok=True)
-    out_path = f"{OUT_DIR}/{model_path}_eval_request_{private}_{precision}_{weight_type}.json"
+    if revision == "main":
+        out_path = f"{OUT_DIR}/{model_path}_eval_request_{private}_{precision}_{weight_type}.json"
+    else:
+        rev_name = revision
+        if "rev_name" > 30:
+            rev_name = rev_name[:7]
+        out_path = f"{OUT_DIR}/{model_path}_eval_request_{rev_name}_{private}_{precision}_{weight_type}.json"
 
     with open(out_path, "w", encoding="utf-8") as f:
         json.dump(eval_entry, f, indent=4, ensure_ascii=False)