Code cleanup

app.py

@@ -1,7 +1,6 @@
 import gradio as gr
 import pandas as pd
 from apscheduler.schedulers.background import BackgroundScheduler
-#from huggingface_hub import snapshot_download
 import re
 import plotly.graph_objects as go
 
@@ -36,35 +35,18 @@ from src.leaderboard.read_evals import get_model_answers_html_file
 skills = ['MMLU', 'General Knowledge', 'Reasoning & Math', 'Translation (incl Dialects)', 'Trust & Safety', 'Writing (incl Dialects)', 'RAG QA', 'Reading Comprehension', 'Arabic Language & Grammar', 'Diacritization', 'Dialect Detection', 'Sentiment Analysis', 'Summarization', 'Instruction Following', 'Transliteration', 'Paraphrasing', 'Entity Extraction', 'Long Context', 'Coding', 'Hallucination', 'Function Calling', 'Structuring']
 
 
-def restart_space():
-    API.restart_space(repo_id=REPO_ID)
-
-### Space initialisation
-"""
-try:
-    print(EVAL_REQUESTS_PATH)
-    snapshot_download(
-        repo_id=QUEUE_REPO, local_dir=EVAL_REQUESTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN
-    )
-except Exception:
-    restart_space()
-try:
-    print(EVAL_RESULTS_PATH)
-    snapshot_download(
-        repo_id=RESULTS_REPO, local_dir=EVAL_RESULTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN
-    )
-except Exception:
-    restart_space()
-"""
-
 LEADERBOARD_DF = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, COLS, BENCHMARK_COLS)
 
 (
     finished_eval_queue_df,
-    running_eval_queue_df,
     pending_eval_queue_df,
 ) = get_evaluation_queue_df(EVAL_REQUESTS_PATH, EVAL_COLS)
 
+
+def restart_space():
+    API.restart_space(repo_id=REPO_ID)
+
+
 def hide_skill_columns(dataframe, exceptions=[]):
     return dataframe[[c.name for c in fields(AutoEvalColumn) if c.displayed_by_default or c.name in exceptions]]
 
@@ -115,9 +97,6 @@ def init_leaderboard(dataframe):
 def init_skill_leaderboard(dataframe):
 
     
-
-    ## create selector for model skills, based on the selector filter the dataframe
-
     skills_dropdown = gr.Dropdown(choices=skills, label="Select Skill", value=skills[0])
 
     def filter_dataframe(skill):
@@ -126,6 +105,7 @@ def init_skill_leaderboard(dataframe):
         new_skill_name = skill+" Score"
         filtered_df.rename(columns={skill: new_skill_name}, inplace=True)
         filtered_df[new_skill_name] = make_column_bold(filtered_df[new_skill_name])
+
         ## reorder columns of filtered_df and insert skill in the middle
         filtered_df = filtered_df[list(filtered_df.columns[:4]) + [new_skill_name] + list(filtered_df.columns[4:-1])]
         filtered_df["Rank"] = range(1, len(filtered_df) + 1)
@@ -143,6 +123,7 @@ def init_skill_leaderboard(dataframe):
 
     leaderboard_by_skill = filter_dataframe(skills[0])
     skills_dropdown.change(filter_dataframe, inputs=skills_dropdown, outputs=leaderboard_by_skill)
+
     return leaderboard_by_skill
 
 
@@ -159,13 +140,16 @@ def init_size_leaderboard(dataframe):
     sizes_dropdown = gr.Dropdown(choices=size_names, label="Select Model Size", value=size_names[0])
 
     def filter_dataframe(size_name):
+
         ##map size name to size key
         size_name_mapped_to_key = size_keys[size_names.index(size_name)]
+
         ##slice array from 0 to index of size
         size_list = size_keys[size_keys.index(size_name_mapped_to_key):]
         filtered_df = dataframe[dataframe["Size"].isin(size_list)].reset_index(drop=True)
         filtered_df["Rank"] = range(1, len(filtered_df) + 1)
         styler = perform_cell_formatting(filtered_df)
+
         return gr.Dataframe(
             value=styler,
             datatype="markdown",
@@ -179,19 +163,18 @@ def init_size_leaderboard(dataframe):
 
     leaderboard_by_skill = filter_dataframe(size_names[0])
     sizes_dropdown.change(filter_dataframe, inputs=sizes_dropdown, outputs=leaderboard_by_skill)
+
     return leaderboard_by_skill
 
 def strip_html_tags(model_name):
         return re.sub('<[^<]+?>', '', model_name)
-    
-
 
 def get_model_info_blocks(chosen_model_name):
 
     model_names = LEADERBOARD_DF["Model Name"].unique().tolist()
     model_names_clean = [strip_html_tags(model_name) for model_name in model_names]
-    
     model_name_full = model_names[model_names_clean.index(chosen_model_name)]
+
     filtered_df = LEADERBOARD_DF[LEADERBOARD_DF["Model Name"]==model_name_full].reset_index(drop=True)
     skills_bar_df = pd.DataFrame({
         'Skills': skills,
@@ -258,14 +241,13 @@ def init_compare_tab(dataframe):
 
     model_names = dataframe["Model Name"].unique().tolist()
     model_names_clean = [strip_html_tags(model_name) for model_name in model_names]
+
     with gr.Row():
         models_dropdown = gr.Dropdown(choices=model_names_clean, label="Select Models",
                                        value=model_names_clean[0], multiselect=True)
         
 
     def draw_radar_chart(models):
-        print(models)
-
 
         fig = go.Figure()
 
@@ -310,7 +292,9 @@ def init_compare_tab(dataframe):
 
 
 demo = gr.Blocks(css=custom_css)
+
 with demo:
+
     gr.HTML(TITLE, elem_classes="abl_header")
     gr.HTML(INTRODUCTION_TEXT, elem_classes="abl_desc_text")
 
@@ -329,9 +313,9 @@ with demo:
 
         with gr.TabItem("🔬 Deep Dive", elem_id="llm-benchmark-tab-compare", id=4):
 
-
             model_names = LEADERBOARD_DF["Model Name"].unique().tolist()
             model_names_clean = [strip_html_tags(model_name) for model_name in model_names]
+
             with gr.Row():
                 models_dropdown = gr.Dropdown(choices=model_names_clean, label="Select Model", value=model_names_clean[0])
 
@@ -341,8 +325,10 @@ with demo:
             models_dropdown.change(get_model_info_blocks, inputs=models_dropdown, outputs=[model_name,benchmark_score,rank,speed,contamination,size,skills_bar,answers_html])
   
         with gr.TabItem("🚀 Submit here", elem_id="llm-benchmark-tab-submit", id=5):
+
             with gr.Row():
                 gr.Markdown("# Submit your model", elem_classes="markdown-text")
+
             with gr.Column():
                 gr.Markdown("### Please confirm that you understand and accept the conditions below before submitting your model:")
                 prereqs_checkboxes = gr.CheckboxGroup(["I have successfully run the ABB benchmark script on my model using my own infrastructure, and I am not using the Leaderboard for testing purposes",
@@ -368,6 +354,7 @@ with demo:
                 )
             
             submission_result = gr.Markdown()
+
             submit_button.click(
                 add_new_eval,
                 [
@@ -375,7 +362,9 @@ with demo:
                 ],
                 submission_result,
             )
+
             with gr.Column():
+
                 with gr.Row():
                     gr.Markdown(EVALUATION_QUEUE_TEXT, elem_classes="markdown-text")
 
@@ -391,17 +380,6 @@ with demo:
                                 datatype=EVAL_TYPES,
                                 row_count=5,
                             )
-                    with gr.Accordion(
-                        f"🔄 Running Evaluation Queue ({len(running_eval_queue_df)})",
-                        open=False,
-                    ):
-                        with gr.Row():
-                            running_eval_table = gr.components.Dataframe(
-                                value=running_eval_queue_df,
-                                headers=EVAL_COLS,
-                                datatype=EVAL_TYPES,
-                                row_count=5,
-                            )
 
                     with gr.Accordion(
                         f"⏳ Pending Evaluation Queue ({len(pending_eval_queue_df)})",
@@ -416,6 +394,7 @@ with demo:
                             )
 
         with gr.TabItem("📝 FAQ", elem_id="llm-benchmark-tab-faq", id=6):
+
             gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")
 
     with gr.Row():
@@ -423,7 +402,7 @@ with demo:
             citation_button = gr.Textbox(
                 value=CITATION_BUTTON_TEXT,
                 label=CITATION_BUTTON_LABEL,
-                lines=10,
+                lines=8,
                 elem_id="citation-button",
                 show_copy_button=True,
             )

src/about.py

@@ -38,19 +38,6 @@ class EvalDimensions(Enum):
 
 
 
-
-    
-
-
-
-
-
-
-NUM_FEWSHOT = 0 # Change with your few shot
-# ---------------------------------------------------
-
-
-
 # Your leaderboard name
 TITLE = """<div ><img class='abl_header_image' src='https://huggingface.co/spaces/silma-ai/Arabic-LLM-Broad-Leaderboard/resolve/main/src/images/abl_logo.png' ></div>"""
 
@@ -166,7 +153,6 @@ EVALUATION_QUEUE_TEXT = """
 
 CITATION_BUTTON_LABEL = "Copy the following snippet to cite the Leaderboard"
 CITATION_BUTTON_TEXT = r"""
-
 @misc{ABL,
   author = {SILMA.AI Team},
   title = {Arabic Broad Leaderboard},

src/display/css_html_js.py

@@ -108,13 +108,13 @@ custom_css = """
    
 }
 .abl_header{
-margin:0px auto 0px auto;
+    margin:0px auto 0px auto;
 }
 .abl_header_image{
-margin:0px auto 0px auto;
-width:50%;
-display:block;
-border-radius: 10px;
+    margin:0px auto 0px auto;
+    width:50%;
+    display:block;
+    border-radius: 10px;
 }
 
 .tabs{
@@ -164,11 +164,3 @@ color:unset;
     margin-top:20px;
 }
 """
-
-get_window_url_params = """
-    function(url_params) {
-        const params = new URLSearchParams(window.location.search);
-        url_params = Object.fromEntries(params);
-        return url_params;
-    }
-    """

src/display/formatting.py

@@ -6,11 +6,6 @@ def make_clickable_model(model_name):
     link = f"https://huggingface.co/{model_name}"
     return model_hyperlink(link, model_name)
 
-def make_contamination_red(contamination_score):
-    if contamination_score <=0:    
-        return f"<p class='clean'  style='display:block;background-color:green !important;padding:5px;color: white;  text-align: center;margin:0px'  title='Clean model!'>{round((contamination_score))}</p>"
-    else:
-        return f"<p class='contaminated' style='display:block;background-color:red !important;padding:5px;color: white; text-align: center;margin:0px' title='Contaminated model!'>{round((contamination_score),2)}</p>"
 
 def styled_error(error):
     return f"<p style='color: red; font-size: 20px; text-align: center;'>{error}</p>"
@@ -27,6 +22,3 @@ def styled_message(message):
 def has_no_nan_values(df, columns):
     return df[columns].notna().all(axis=1)
 
-
-def has_nan_values(df, columns):
-    return df[columns].isna().any(axis=1)

src/display/utils.py

@@ -1,8 +1,4 @@
 from dataclasses import dataclass, make_dataclass
-from enum import Enum
-
-import pandas as pd
-
 from src.about import EvalDimensions
 
 def fields(raw_class):
@@ -24,12 +20,8 @@ class ColumnContent:
 auto_eval_column_dict = []
 # Init
 auto_eval_column_dict.append(["rank", ColumnContent, ColumnContent("Rank", "str", True, False)])
-
 auto_eval_column_dict.append(["model_source", ColumnContent, ColumnContent("Source", "str", True, False)])
 auto_eval_column_dict.append(["model_category", ColumnContent, ColumnContent("Size", "str", True, False)])
-
-
-#auto_eval_column_dict.append(["model_type_symbol", ColumnContent, ColumnContent("T", "str", True, never_hidden=True)])
 auto_eval_column_dict.append(["model", ColumnContent, ColumnContent("Model Name", "markdown", True, never_hidden=True)])
 #Scores
 auto_eval_column_dict.append(["average_score", ColumnContent, ColumnContent("Benchmark Score (0-10)", "number", True)])
@@ -38,17 +30,6 @@ for eval_dim in EvalDimensions:
          auto_eval_column_dict.append([eval_dim.name, ColumnContent, ColumnContent(eval_dim.value.col_name, "number", True)])
     else:
         auto_eval_column_dict.append([eval_dim.name, ColumnContent, ColumnContent(eval_dim.value.col_name, "number", False)])
-# Model information
-
-#auto_eval_column_dict.append(["model_type", ColumnContent, ColumnContent("Type", "str", False)])
-#auto_eval_column_dict.append(["architecture", ColumnContent, ColumnContent("Architecture", "str", False)])
-#auto_eval_column_dict.append(["weight_type", ColumnContent, ColumnContent("Weight type", "str", False, True)])
-#auto_eval_column_dict.append(["precision", ColumnContent, ColumnContent("Precision", "str", False)])
-#auto_eval_column_dict.append(["license", ColumnContent, ColumnContent("License", "str", False)])
-#auto_eval_column_dict.append(["params", ColumnContent, ColumnContent("#Params (B)", "number", False)])
-#auto_eval_column_dict.append(["likes", ColumnContent, ColumnContent("Popularity (Likes)", "number", False)])
-#auto_eval_column_dict.append(["still_on_hub", ColumnContent, ColumnContent("Available on the hub", "bool", False)])
-#auto_eval_column_dict.append(["revision", ColumnContent, ColumnContent("Model sha", "str", False, False)])
 
 
 # We use make dataclass to dynamically fill the scores from Tasks
@@ -59,60 +40,8 @@ AutoEvalColumn = make_dataclass("AutoEvalColumn", auto_eval_column_dict, frozen=
 class EvalQueueColumn:  # Queue column
     model = ColumnContent("model", "markdown", True)
     revision = ColumnContent("revision", "str", True)
-    #private = ColumnContent("private", "bool", True)
-    #precision = ColumnContent("precision", "str", True)
-    #weight_type = ColumnContent("weight_type", "str", "Original")
     status = ColumnContent("status", "str", True)
 
-## All the model information that we might need
-@dataclass
-class ModelDetails:
-    name: str
-    display_name: str = ""
-    symbol: str = "" # emoji
-
-"""    
-class ModelType(Enum):
-
-
-    PT = ModelDetails(name="pretrained", symbol="🟢")
-    FT = ModelDetails(name="fine-tuned", symbol="🔶")
-    IFT = ModelDetails(name="instruction-tuned", symbol="⭕")
-    RL = ModelDetails(name="RL-tuned", symbol="🟦")
-    Unknown = ModelDetails(name="", symbol="?")
-
-    def to_str(self, separator=" "):
-        return f"{self.value.symbol}{separator}{self.value.name}"
-
-    @staticmethod
-    def from_str(type):
-        if "fine-tuned" in type or "🔶" in type:
-            return ModelType.FT
-        if "pretrained" in type or "🟢" in type:
-            return ModelType.PT
-        if "RL-tuned" in type or "🟦" in type:
-            return ModelType.RL
-        if "instruction-tuned" in type or "⭕" in type:
-            return ModelType.IFT
-        return ModelType.Unknown
-
-class WeightType(Enum):
-    Adapter = ModelDetails("Adapter")
-    Original = ModelDetails("Original")
-    Delta = ModelDetails("Delta")
-
-class Precision(Enum):
-    float16 = ModelDetails("float16")
-    bfloat16 = ModelDetails("bfloat16")
-    Unknown = ModelDetails("?")
-
-    def from_str(precision):
-        if precision in ["torch.float16", "float16"]:
-            return Precision.float16
-        if precision in ["torch.bfloat16", "bfloat16"]:
-            return Precision.bfloat16
-        return Precision.Unknown
-"""
 # Column selection
 COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden]
 

src/envs.py

@@ -19,7 +19,5 @@ CACHE_PATH=os.getenv("HF_HOME", ".")
 # Local caches
 EVAL_REQUESTS_PATH = os.path.join(CACHE_PATH, "requests")
 EVAL_RESULTS_PATH = os.path.join(CACHE_PATH, "results")
-EVAL_REQUESTS_PATH_BACKEND = os.path.join(CACHE_PATH, "eval-queue-bk")
-EVAL_RESULTS_PATH_BACKEND = os.path.join(CACHE_PATH, "eval-results-bk")
 
 API = HfApi(token=TOKEN)

src/leaderboard/read_evals.py

@@ -1,14 +1,11 @@
 import glob
 import json
-import math
 import os
 from dataclasses import dataclass
-
 import dateutil
-import numpy as np
 
-from src.display.formatting import make_clickable_model, make_contamination_red
-from src.display.utils import AutoEvalColumn, EvalDimensions#, ModelType, Precision, WeightType
+from src.display.formatting import make_clickable_model
+from src.display.utils import AutoEvalColumn, EvalDimensions
 from src.submission.check_validity import is_model_on_hub
 
 
@@ -20,17 +17,9 @@ class EvalResult:
     full_model: str # org/model (path on hub)
     org: str 
     model: str
-    #revision: str # commit hash, "" if main
     results: dict
-    #precision: Precision = Precision.Unknown
-    #model_type: ModelType = ModelType.Unknown # Pretrained, fine tuned, ...
     model_source: str = "" # HF, API, ...
     model_category: str = "" #Nano, Small, Medium, Large
-    #weight_type: WeightType = WeightType.Original # Original or Adapter
-    #architecture: str = "Unknown" 
-    #license: str = "?"
-    #likes: int = 0
-    #num_params: int = 0
     date: str = "" # submission date of request file
     still_on_hub: bool = False
 
@@ -42,41 +31,31 @@ class EvalResult:
 
         config = data.get("config")
 
-        # Precision
-        #precision = Precision.from_str(config.get("model_dtype"))
-
         # Get model and org
         org_and_model = config.get("model", config.get("model_args", None))
-        print("******* org_and_model **********", config)
+  
         org_and_model = org_and_model.split("/", 1)
 
         if len(org_and_model) == 1:
             org = None
             model = org_and_model[0]
-            result_key = f"{model}"#_{precision.value.name}
+            result_key = f"{model}"
         else:
             org = org_and_model[0]
             model = org_and_model[1]
-            result_key = f"{org}_{model}"#_{precision.value.name}
+            result_key = f"{org}_{model}"
         full_model = "/".join(org_and_model)
 
-        still_on_hub, _, model_config = is_model_on_hub(
+        still_on_hub, _, _ = is_model_on_hub(
             full_model, config.get("model_sha", "main"), trust_remote_code=True, test_tokenizer=False
         )
         
-        """
-        architecture = "?"
-        if model_config is not None:
-            architectures = getattr(model_config, "architectures", None)
-            if architectures:
-                architecture = ";".join(architectures)
-        """
 
         # Extract results available in this file (some results are split in several files)
         results = {}
 
         results_obj = data.get("results")
-        print(results_obj)
+        
         results["average_score"] = results_obj.get("average_score")
         results["speed"] = results_obj.get("speed")
         results["contamination_score"] = results_obj.get("contamination_score")
@@ -98,50 +77,30 @@ class EvalResult:
             model=model,
             model_source=config.get("model_source", ""),
             model_category=config.get("model_category", ""),
-            #num_params=config.get("params", 0),
-            #license=config.get("license", "?"),
-            #likes=config.get("likes", -1),
             results=results,
-            #precision=precision,  
-            #revision= config.get("model_sha", ""),
             still_on_hub=still_on_hub,
-            #architecture=architecture
         )
 
     def update_with_request_file(self, requests_path):
         """Finds the relevant request file for the current model and updates info with it"""
-        request_file = get_request_file_for_model(requests_path, self.full_model) #, self.precision.value.name
+        request_file = get_request_file_for_model(requests_path, self.full_model)
         try:
             with open(request_file, "r") as f:
                 request = json.load(f)
             
-            #self.model_type = ModelType.from_str(request.get("model_type", ""))
-            #self.weight_type = WeightType[request.get("weight_type", "Original")]
-            #self.license = request.get("license", "?")
-            #self.likes = request.get("likes", 0)
-            #self.params = request.get("params", 0)
             self.date = request.get("submitted_time", "")
         except Exception:
-            print(f"Could not find request file for {self.org}/{self.model}") # with precision {self.precision.value.name}
+            print(f"Could not find request file for {self.org}/{self.model}")
 
     def to_dict(self):
         """Converts the Eval Result to a dict compatible with our dataframe display"""
         average_score = self.results["average_score"]
         data_dict = {
             "eval_name": self.eval_name,  # not a column, just a save name,
-            #AutoEvalColumn.precision.name: self.precision.value.name,
             AutoEvalColumn.model_source.name: self.model_source,
             AutoEvalColumn.model_category.name: self.model_category,
-            #AutoEvalColumn.model_type_symbol.name: self.model_type.value.symbol,
-            #AutoEvalColumn.weight_type.name: self.weight_type.value.name,
-            #AutoEvalColumn.architecture.name: self.architecture,
             AutoEvalColumn.model.name: make_clickable_model(self.full_model),
-            #AutoEvalColumn.revision.name: self.revision,
             AutoEvalColumn.average_score.name: average_score,
-            #AutoEvalColumn.license.name: self.license,
-            #AutoEvalColumn.likes.name: self.likes,
-            #AutoEvalColumn.params.name: self.num_params,
-            #AutoEvalColumn.still_on_hub.name: self.still_on_hub,
         }
 
         for eval_dim in EvalDimensions:
@@ -159,11 +118,11 @@ class EvalResult:
         return data_dict
 
 
-def get_request_file_for_model(requests_path, model_name): #,precision
+def get_request_file_for_model(requests_path, model_name):
     """Selects the correct request file for a given model. Only keeps runs tagged as FINISHED"""
     request_files = os.path.join(
         requests_path,
-        f"{model_name}_eval_request_*.json",
+        f"{model_name}_eval_request.json",
     )
     
     request_files = glob.glob(request_files)
@@ -176,7 +135,6 @@ def get_request_file_for_model(requests_path, model_name): #,precision
             req_content = json.load(f)
             if (
                 req_content["status"] in ["FINISHED"]
-                #and req_content["precision"] == precision.split(".")[-1]
             ):
                 request_file = tmp_request_file
     return request_file
@@ -187,8 +145,8 @@ def get_raw_eval_results(results_path: str, requests_path: str) -> list[EvalResu
     model_result_filepaths = []
 
     for root, _, files in os.walk(results_path):
-        print("HERE",files)
-        # We should only have json files in model results ##we allow HTML files
+        
+        ## we allow HTML files now
         #if len(files) == 0 or any([not f.endswith(".json") for f in files]):
         #    continue
         files = [f for f in files if f.endswith(".json")]
@@ -199,7 +157,7 @@ def get_raw_eval_results(results_path: str, requests_path: str) -> list[EvalResu
         except dateutil.parser._parser.ParserError as e:
             print("Error",e)
             files = [files[-1]]
-        print(files)
+        
         for file in files:
             model_result_filepaths.append(os.path.join(root, file))
 
@@ -207,7 +165,7 @@ def get_raw_eval_results(results_path: str, requests_path: str) -> list[EvalResu
     for model_result_filepath in model_result_filepaths:
         # Creation of result
         eval_result = EvalResult.init_from_json_file(model_result_filepath)
-        eval_result.update_with_request_file(requests_path)
+        #eval_result.update_with_request_file(requests_path) ##not needed, save processing time
 
         # Store results of same eval together
         eval_name = eval_result.eval_name
@@ -217,20 +175,17 @@ def get_raw_eval_results(results_path: str, requests_path: str) -> list[EvalResu
             eval_results[eval_name] = eval_result
 
     results = []
-    #print(eval_results.values())
+
     for v in eval_results.values():
         try:
-            print(v.to_dict())
+
             v.to_dict() # we test if the dict version is complete
             results.append(v)
         except KeyError:  # not all eval values present
             print("Key error in eval result, skipping")
             
-            print(v)
-            print(v.to_dict())
             continue
 
-    print(results)
     return results
 
 

src/populate.py

@@ -31,12 +31,7 @@ def get_leaderboard_df(results_path: str, requests_path: str, cols: list, benchm
                 df[col] = df[col].round(2)
 
         df["Benchmark Score (0-10)"] = df["Benchmark Score (0-10)"].astype(str)
-        print(df["Benchmark Score (0-10)"])
 
-        print("###############\n\n\n\n\n\n###############")
-
-        print(df)
-        print(df.info())
 
 
         return df
@@ -64,10 +59,10 @@ def get_evaluation_queue_df(save_path: str, cols: list) -> list[pd.DataFrame]:
             # this is a folder
 
             sub_entries = [e for e in os.listdir(f"{save_path}/{entry}") if os.path.isfile(f"{save_path}/{entry}/{e}") ]#and not e.startswith(".")
-            print(f"Sub entries: {sub_entries}")
+            
             for sub_entry in sub_entries:
                 file_path = os.path.join(save_path, entry, sub_entry)
-                print(f"{file_path}")
+                
 
                 with open(file_path) as fp:
                     data = json.load(fp)
@@ -78,10 +73,8 @@ def get_evaluation_queue_df(save_path: str, cols: list) -> list[pd.DataFrame]:
     
 
     pending_list = [e for e in all_evals if e["status"] in ["PENDING", "RERUN"]]
-    print(pending_list)
-    running_list = [e for e in all_evals if e["status"] == "RUNNING"]
+
     finished_list = [e for e in all_evals if e["status"].startswith("FINISHED") or e["status"] == "PENDING_NEW_EVAL"]
     df_pending = pd.DataFrame.from_records(pending_list, columns=cols)
-    df_running = pd.DataFrame.from_records(running_list, columns=cols)
     df_finished = pd.DataFrame.from_records(finished_list, columns=cols)
-    return df_finished[cols], df_running[cols], df_pending[cols]
+    return df_finished[cols], df_pending[cols]

src/submission/check_validity.py

@@ -65,7 +65,7 @@ def get_model_size(model_info: ModelInfo): #, precision: str
         model_size = round(model_info.safetensors["total"] / 1e9, 3)
     except (AttributeError, TypeError):
         return 0  # Unknown model sizes are indicated as 0, see NUMERIC_INTERVALS in app.py
-    print("******* model size **********",model_size)
+
     size_factor = 1#8 if (precision == "GPTQ" or "gptq" in model_info.modelId.lower()) else 1
     model_size = size_factor * model_size
     return model_size
@@ -88,7 +88,7 @@ def already_submitted_models(requested_models_dir: str) -> set[str]:
                     continue
                 with open(os.path.join(root, file), "r") as f:
                     info = json.load(f)
-                    file_names.append(f"{info['model']}")#_{info['revision']}_{info['precision']}
+                    file_names.append(f"{info['model']}")
 
                     # Select organisation
                     if info["model"].count("/") == 0 or "submitted_time" not in info:

src/submission/submit.py

@@ -18,11 +18,6 @@ USERS_TO_SUBMISSION_DATES = None
 def add_new_eval(
     model: str,
     progress=gr.Progress()
-    #base_model: str,
-    #revision: str,
-    #precision: str,
-    #weight_type: str,
-    #model_type: str,
 ):
     global REQUESTED_MODELS
     global USERS_TO_SUBMISSION_DATES
@@ -37,18 +32,19 @@ def add_new_eval(
         user_name = model.split("/")[0]
         model_path = model.split("/")[1]
 
-    #precision = precision.split(" ")[0]
     current_time = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
 
 
     progress(0.1, desc=f"Checking model {model} on hub")
 
-    if not is_model_on_hub(model_name=model, token=TOKEN, test_tokenizer=True): #revision=revision
+    if not is_model_on_hub(model_name=model, token=TOKEN, test_tokenizer=True):
         yield styled_error("Model does not exist on HF Hub. Please select a valid model name.")
         return
 
-    ##check for org banning
+
     progress(0.2, desc=f"Checking for banned orgs")
+    
+    ##check for org banning
     banned_orgs = [{
         'org_name':'TEMPLATE',
         'banning_reason':'Submitting contaminated models'
@@ -60,34 +56,16 @@ def add_new_eval(
         )   
         return
 
-    """
-    if model_type is None or model_type == "":
-        return styled_error("Please select a model type.")
-
-    # Does the model actually exist?
-    if revision == "":
-        revision = "main"
-
-    # Is the model on the hub?
-    if weight_type in ["Delta", "Adapter"]:
-        base_model_on_hub, error, _ = is_model_on_hub(model_name=base_model, revision=revision, token=TOKEN, test_tokenizer=True)
-        if not base_model_on_hub:
-            return styled_error(f'Base model "{base_model}" {error}')
-
-    if not weight_type == "Adapter":
-        model_on_hub, error, _ = is_model_on_hub(model_name=model, revision=revision, token=TOKEN, test_tokenizer=True)
-        if not model_on_hub:
-            return styled_error(f'Model "{model}" {error}')
-    """
+   
     # Is the model info correctly filled?
     try:
-        model_info = API.model_info(repo_id=model)#, revision=revision
+        model_info = API.model_info(repo_id=model)
     except Exception:
         yield styled_error("Could not get your model information. Please fill it up properly.")
         return
     
     progress(0.3, desc=f"Checking model size")
-    model_size = get_model_size(model_info=model_info)#, precision=precision
+    model_size = get_model_size(model_info=model_info)
 
     if model_size>15:
         yield styled_error("We currently accept community-submitted models up to 15 billion parameters only. If you represent an organization then please contact us at benchmark@silma.ai")
@@ -131,7 +109,7 @@ def add_new_eval(
     progress(0.8, desc=f"Checking same model submissions")
 
     # Check for duplicate submission
-    if f"{model}" in REQUESTED_MODELS: #_{revision}_{precision}
+    if f"{model}" in REQUESTED_MODELS:
         yield styled_warning("This model has already been submitted.")
         return
 
@@ -141,17 +119,11 @@ def add_new_eval(
     eval_entry = {
         "model": model,
         "model_sha": model_info.sha,
-        #"base_model": base_model,
-        #"revision": revision,
-        #"precision": precision,
-        #"weight_type": weight_type,
         "status": "PENDING",
         "submitted_time": current_time,
-        #"model_type": model_type,
         "likes": model_info.likes,
         "params": model_size,
         "license": license,
-        #"private": False,
     }
 
 
@@ -160,7 +132,7 @@ def add_new_eval(
     print("Creating eval file")
     OUT_DIR = f"{EVAL_REQUESTS_PATH}/{user_name}"
     os.makedirs(OUT_DIR, exist_ok=True)
-    out_path = f"{OUT_DIR}/{model_path}_eval_request.json" #_{precision}_{weight_type}
+    out_path = f"{OUT_DIR}/{model_path}_eval_request.json"
 
     with open(out_path, "w") as f:
         f.write(json.dumps(eval_entry))
@@ -182,7 +154,7 @@ def add_new_eval(
         queue_data = json.load(f)
 
     queue_len = len(queue_data)
-    print(f"Queue length: {queue_len}")
+    
 
     if queue_len == 0:
         queue_data = []
@@ -192,10 +164,6 @@ def add_new_eval(
     
     queue_data.append(eval_entry)
 
-    print(queue_data)
-
-    #with open(queue_file, "w") as f:
-    #    json.dump(queue_data, f)
 
     print("Updating eval queue file")
     API.upload_file(