Updated system to connect the different repos

app.py

@@ -1,40 +1,57 @@
 import os
+import json
+import datetime
 from email.utils import parseaddr
 
 import gradio as gr
 import pandas as pd
+import numpy as np
 
 from datasets import load_dataset
 from apscheduler.schedulers.background import BackgroundScheduler
 from huggingface_hub import HfApi
 
 # InfoStrings
-from content import *
+from scorer import question_scorer
+from content import format_warning, format_log, TITLE, INTRODUCTION_TEXT, CHANGELOG_TEXT, CITATION_BUTTON_LABEL, CITATION_BUTTON_TEXT
 
 BALM_TOKEN = os.environ.get("BALM_TOKEN", None)
-owner="balm" # change to balm once possible
 
+OWNER="balm"
+SUBMISSION_DATASET = f"{OWNER}/submissions"
+
+SPLIT="validation" #Change to test once we are ready to go 
 api = HfApi()
 
+os.makedirs("scored", exist_ok=True)
+
+# Display the results
 eval_results = {}
 for level in range(1, 4):
-    eval_results[level] = load_dataset(f"{owner}/BALM_ResultsLevel{level}", use_auth_token=BALM_TOKEN, split="dev")
+    eval_results[level] = load_dataset(f"{OWNER}/BALM_ResultsLevel{level}", token=BALM_TOKEN, split=SPLIT)
 
 eval_dataframe_1 = pd.DataFrame(eval_results[1].remove_columns("mail"))
 eval_dataframe_2 = pd.DataFrame(eval_results[2].remove_columns("mail"))
 eval_dataframe_3 = pd.DataFrame(eval_results[3].remove_columns("mail"))
 
+# Gold answers
+gold_results = {}
+for level in range(1, 4):
+    level_dataset = load_dataset(f"{OWNER}/BALM", f"2023_level{level}", split=SPLIT, token=BALM_TOKEN)
+    gold_results[level] = {row["task_id"]: row["ground_truth"] for row in level_dataset}
+
+
 def restart_space():
-    api.restart_space(repo_id=f"{owner}/BALM_Leaderboard", token=BALM_TOKEN)
+    api.restart_space(repo_id=f"{OWNER}/BALM_Leaderboard", token=BALM_TOKEN)
 
 
-COLS = ["Model", "Organisation", "Reported accuracy ⬆️"]
-TYPES = ["str", "str", "number",]
+COLS = ["Model", "Score ⬆️", "Organisation"]
+TYPES = ["str", "number", "str",]
 
 def add_new_eval(
     level_of_dev: str,
     model: str,
-    score: float,
+    path_to_file,
     organisation: str,
     mail: str,
 ):
@@ -43,68 +60,86 @@ def add_new_eval(
     # Very basic email parsing
     _, parsed_mail = parseaddr(mail)
     if not "@" in parsed_mail:
-        valid_mail = "Please provide a valid email adress."
-        return f"<p style='color: orange; font-size: 20px; text-align: center;'>{valid_mail}</p>"
+        return format_warning("Please provide a valid email adress.")
 
     print("Adding new eval")
 
     # Check if the combination model/org already exists and prints a warning message if yes
     if model.lower() in set(eval_results[level]["model"]) and organisation.lower() in set(eval_results[level]["organisation"]):
-        duplicate_request_message = "This model has been already submitted."
-        return f"<p style='color: orange; font-size: 20px; text-align: center;'>{duplicate_request_message}</p>"
+        return format_warning("This model has been already submitted.")
+
+    # Save submitted file
+    api.upload_file(
+        repo_id=SUBMISSION_DATASET, 
+        path_or_fileobj=path_to_file.name, 
+        path_in_repo=f"{organisation}/{model}/level{level}_raw_{datetime.datetime.today()}.jsonl",
+        repo_type="dataset", 
+        token=BALM_TOKEN
+    )
+
+    # Compute score
+    file_path = path_to_file.name
+    total_score = 0
+    with open(f"scored/{organisation}_{model}.jsonl", "w") as scored_file:
+        with open(file_path, 'r') as f:
+            for line in f:
+                task = json.loads(line)
+
+                if "model_answer" not in task:
+                    raise Exception("No model_answer key in the file provided")
+                answer = task["model_answer"]
+                task_id = task["task_id"]
+                
+                score = question_scorer(task['model_answer'], gold_results[level][task_id])
+                
+                scored_file.write(
+                    json.dumps({
+                        "id": task_id,
+                        "model_answer": answer,
+                        "score": score
+                    }) + "\n"
+                )
+
+                total_score += score
+    
+    # Save scored file
+    api.upload_file(
+        repo_id=SUBMISSION_DATASET, 
+        path_or_fileobj=f"scored/{organisation}_{model}.jsonl",
+        path_in_repo=f"{organisation}/{model}/level{level}_scored_{datetime.datetime.today()}.jsonl", 
+        repo_type="dataset", 
+        token=BALM_TOKEN
+    )
 
     # Actual submission
     eval_entry = {
         "model": model,
-        "score": score,
+        "score": total_score,
         "organisation": organisation,
         "mail": mail,
     }
     eval_results[level] = eval_results[level].add_item(eval_entry)
-    eval_results[level].push_to_hub(f"{owner}/BALM_ResultsLevel{level}", token=BALM_TOKEN, split="dev")
+    # TODO: change split to "test" once we have the actual results 
+    eval_results[level].push_to_hub(f"{OWNER}/BALM_ResultsLevel{level}", token=BALM_TOKEN, split=SPLIT)
 
-    success_message = f"Model {model} submitted by {organisation}."
-    return f"<p style='color: green; font-size: 20px; text-align: center;'>{success_message}</p>"
+    return format_log(f"Model {model} submitted by {organisation} successfully. \nPlease refresh the leaderboard, and wait for up to an hour to see the score displayed")
 
 
 def refresh():
     eval_results = {}
     for level in range(1, 4):
-        eval_results[level] = load_dataset(f"{owner}/BALM_ResultsLevel{level}", use_auth_token=BALM_TOKEN, split="dev")
+        eval_results[level] = load_dataset(f"{OWNER}/BALM_ResultsLevel{level}", use_auth_token=BALM_TOKEN, split=SPLIT)
     eval_dataframe_1 = pd.DataFrame(eval_results[1].remove_columns("mail"))
     eval_dataframe_2 = pd.DataFrame(eval_results[2].remove_columns("mail"))
     eval_dataframe_3 = pd.DataFrame(eval_results[3].remove_columns("mail"))
     return eval_dataframe_1, eval_dataframe_2, eval_dataframe_3
 
+def upload_file(files):
+    file_paths = [file.name for file in files]
+    return file_paths
 
-custom_css = """
-#changelog-text {
-    font-size: 16px !important;
-}
-
-#changelog-text h2 {
-    font-size: 18px !important;
-}
 
-.markdown-text {
-    font-size: 16px !important;
-}
-
-#citation-button span {
-    font-size: 16px !important;
-}
-
-#citation-button textarea {
-    font-size: 16px !important;
-}
-
-#citation-button > label > button {
-    margin: 6px;
-    transform: scale(1.3);
-}
-"""
-
-demo = gr.Blocks(css=custom_css)
+demo = gr.Blocks()
 with demo:
     gr.HTML(TITLE)
     gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
@@ -122,26 +157,17 @@ with demo:
                 changelog = gr.Markdown(CHANGELOG_TEXT, elem_id="changelog-text")
 
     with gr.Tab("Results: Level 1"):
-        with gr.Tab("Results on Dev Set"):
-            leaderboard_table_1 = gr.components.Dataframe(
-                value=eval_dataframe_1, headers=COLS, datatype=TYPES, max_rows=20
-            )
-        with gr.Tab("Results on Test Set"):
-            gr.Textbox(label="Info", value="The test set is currently private! Come back when performances on the dev set increased!")
+        leaderboard_table_1 = gr.components.Dataframe(
+            value=eval_dataframe_1, headers=COLS, datatype=TYPES, interactive=False,
+        )
     with gr.Tab("Results: Level 2"):
-        with gr.Tab("Results on Dev Set"):
-            leaderboard_table_2 = gr.components.Dataframe(
-                value=eval_dataframe_2, headers=COLS, datatype=TYPES, max_rows=20
-            )
-        with gr.Tab("Results on Test Set"):
-            gr.Textbox(label="Info", value="The test set is currently private! Come back when performances on the dev set increased!")
+        leaderboard_table_2 = gr.components.Dataframe(
+            value=eval_dataframe_2, headers=COLS, datatype=TYPES, interactive=False,
+        )
     with gr.Tab("Results: Level 3"):
-        with gr.Tab("Results on Dev Set"):
-            leaderboard_table_3 = gr.components.Dataframe(
-                value=eval_dataframe_3, headers=COLS, datatype=TYPES, max_rows=20
-            )
-        with gr.Tab("Results on Test Set"):
-            gr.Textbox(label="Info", value="The test set is currently private! Come back when performances on the dev set increased!")
+        leaderboard_table_3 = gr.components.Dataframe(
+            value=eval_dataframe_3, headers=COLS, datatype=TYPES, interactive=False,
+        )
 
     refresh_button = gr.Button("Refresh")
     refresh_button.click(
@@ -153,13 +179,12 @@ with demo:
             leaderboard_table_3,
         ],
     )
-
     with gr.Accordion("Submit a new model for evaluation"):
         with gr.Row():
             with gr.Column():
-                level_of_dev = gr.Radio(["Level 1", "Level 2", "Level 3"], value="Level 1", label="Dev set")
+                level_of_test = gr.Radio(["Level 1", "Level 2", "Level 3"], value="Level 1", label="{split} set level")
                 model_name_textbox = gr.Textbox(label="Model name")
-                score = gr.Textbox(label="Score")
+                file_output = gr.File()
             with gr.Column():
                 organisation = gr.Textbox(label="Organisation")
                 mail = gr.Textbox(label="Contact email")
@@ -169,9 +194,9 @@ with demo:
         submit_button.click(
             add_new_eval,
             [
-                level_of_dev,
+                level_of_test,
                 model_name_textbox,
-                score,
+                file_output,
                 organisation,
                 mail
             ],

content.py

@@ -27,6 +27,12 @@ CITATION_BUTTON_TEXT = r"""@misc{balm, # TODO
   title = {Benchmark for Augmented Language Models},
   year = {2023},
   #publisher = {Hugging Face},
-  #howpublished = "\url{https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard}"
+  #howpublished = "\url{https://huggingface.co/spaces/balm/}"
 }"""
 
+
+def format_warning(msg):
+    return f"<p style='color: orange; font-size: 20px; text-align: center;'>{msg}</p>"
+
+def format_log(msg):
+    return f"<p style='color: green; font-size: 20px; text-align: center;'>{msg}</p>"

scorer.py

@@ -0,0 +1,81 @@
+import json
+import re
+import string
+
+import numpy as np
+
+def normalize_text(text: str) -> str:
+    "From QuAC"
+    def remove_articles(text: str) -> str:
+        return re.sub(r"\b(a|an|the)\b", " ", text)
+
+    def white_space_fix(text: str) -> str:
+        return " ".join(text.split())
+
+    def homogeneize_numbers(text: str) -> str:
+        try:
+            return str(float(text))
+        except ValueError:
+            return text
+
+    def remove_punc(text: str) -> str:
+        exclude = set(string.punctuation)
+        return "".join(ch for ch in text if ch not in exclude)
+    
+    def remove_punc2(text: str) -> str:
+        "From Grégoire's code, removes all punctuation, nicer than remove_punc"
+        translator = str.maketrans('', '', string.punctuation)
+        return text.translate(translator)
+
+    def lower(text: str) -> str:
+        return text.lower()
+
+    def _tokenize(text):
+        return re.split(" ", text)
+
+    tokens = [white_space_fix(remove_articles(homogeneize_numbers(remove_punc2(lower(t))))) for t in _tokenize(text)]
+    return " ".join([t for t in tokens if t != ""]).strip()
+
+def extract_answer(input_str: str, prompt_sep: str = 'FINAL ANSWER: ') -> str:
+    answer = input_str.split(prompt_sep)[-1].strip()
+    return answer
+
+def extract_bow(input_str: str) -> list[str]:
+    return input_str.split(" ")
+
+def numbers_equals_in_bow(gold_list: list, pred_list: list) -> bool:
+    # Numbers in prediction bag of words
+    pred_numbers = []
+    for text in pred_list:
+        try:
+            pred_numbers.append(str(float(text)))
+        except ValueError:
+            continue
+
+    for text in gold_list:
+        try:
+            number = str(float(text))
+            if number not in pred_numbers:
+                return False
+        except ValueError:
+            continue
+
+    return True
+
+def affix_quasi_exact_match(gold: str, pred: str) -> float:
+    if not pred:
+        return 0
+
+    normalized_pred = normalize_text(pred)
+    normalized_gold = normalize_text(gold)
+    bow_pred = extract_bow(pred)
+    bow_gold = extract_bow(gold)
+
+    if normalized_pred.startswith(normalized_gold) or normalized_pred.endswith(normalized_gold):
+        if numbers_equals_in_bow(bow_gold, bow_pred):
+            return 1
+    
+    return 0
+
+def question_scorer(gold: str, pred: str) -> float:
+    return affix_quasi_exact_match(gold, pred)