add Dataset Class for management of requests

.gitignore

@@ -7,6 +7,7 @@ __pycache__/
 *ipynb
 .vscode/
 
+hf_cache/
 eval-queue/
 eval-results/
 eval-queue-bk/

app.py

@@ -3,7 +3,6 @@ ESPnet Leaderboard - A Gradio-based leaderboard with multiple tabs and paginatio
 """
 from dotenv import load_dotenv
 import gradio as gr
-import os
 
 from espn_ldbd.display.css_html_js import CUSTOM_CSS
 from espn_ldbd.display.tabs import (
@@ -40,11 +39,17 @@ def create_app():
         # Create tabs for different tasks
         with gr.Tabs():
             for task in leaderboard_ds.get_tasks:
+                print(f"[App] Making Tab for task: {task["task_id"]}")
                 with gr.Tab(task["task_title"]):
-                    create_leaderboard_tab(task["task_title"], leaderboard_data, rows_per_page=30)
+                    create_leaderboard_tab(
+                        task["task_title"],
+                        leaderboard_ds.get_subtasks(task["task_id"]),
+                        leaderboard_data,
+                        rows_per_page=30
+                    )
 
             with gr.Tab("Request a model"):
-                create_submit_tab()
+                create_submit_tab(leaderboard_ds)
 
         # Footer
         gr.HTML("""
@@ -62,6 +67,7 @@ if __name__ == "__main__":
     # Initialize leaderboard data manager
     leaderboard_data = LeaderboardData()
     leaderboard_ds = LeaderboardDataset()
+    leaderboard_ds.start_register_submission(minutes=30)
 
     # Launch app
     app = create_app()

espn_ldbd/display/tabs.py

@@ -2,12 +2,16 @@ import gradio as gr
 import math
 import re
 
-from espn_ldbd.leaderboard.data import LeaderboardData
+from espn_ldbd.leaderboard.data import (
+    LeaderboardData,
+    LeaderboardDataset,
+)
 from espn_ldbd.submission.submit import submit_repo
 
 
 def create_leaderboard_tab(
     task_name: str,
+    sub_tasks: dict,
     leaderboard_data: LeaderboardData,
     rows_per_page: int = 30
 ):
@@ -15,22 +19,34 @@ def create_leaderboard_tab(
 
     # State to track current page
     page_state = gr.State(value=1)
+    selected_subtask = gr.State(value=0)
 
     with gr.Column():
         # Info section
         gr.Markdown(f"## {task_name} Leaderboard")
         gr.Markdown(f"Showing top performing models for {task_name.lower()} task")
-        
-        # Rows per page selector
+
         with gr.Row():
-            rows_dropdown = gr.Dropdown(
-                choices=[10, 20, 30, 50],
-                value=rows_per_page,
-                label="Rows per page",
-                scale=1
-            )
-            refresh_btn = gr.Button("🔄 Refresh Data", scale=1)
-        
+            with gr.Column():
+                subtask_dropdown = gr.Dropdown(
+                    label="Sub Tasks",
+                    choices=[(sub_tasks[x]["sub_task_title"], x) for x in range(len(sub_tasks))],
+                    value=0
+                )
+            with gr.Column():
+                pass
+
+        with gr.Row():
+            with gr.Column():
+                rows_dropdown = gr.Dropdown(
+                    choices=[10, 20, 30, 50],
+                    value=rows_per_page,
+                    label="Rows per page",
+                    scale=1
+                )
+            with gr.Column():
+                refresh_btn = gr.Button("🔄 Refresh Data", scale=1)
+
         # Dataframe display
         dataframe = gr.Dataframe(
             value=leaderboard_data.get_paginated_data(task_name, 1, rows_per_page)[0],
@@ -109,7 +125,8 @@ def create_leaderboard_tab(
     return dataframe, page_info, page_state
 
 
-def create_submit_tab():
+def create_submit_tab(leaderboard_ds: LeaderboardDataset):
+
     def _submit_model(model_text):
         if (model_text is None) or len(model_text) < 1:
             return model_text
@@ -131,11 +148,21 @@ def create_submit_tab():
                 )
                 return model_text
 
-        gr.Info(
-            f"Model id <b>{model_text}</b> submitted.<br/>"
-            "Thank you for your submission."
+        ok_submit, message = leaderboard_ds.submit_repoid(model_text)
+
+        if ok_submit:
+            gr.Info(
+                f"Model id <b>{model_text}</b> submitted.<br/>"
+                "Thank you for your submission."
+            )
+            return ""
+
+        gr.Warning(
+            f"Model id <b>{model_text}</b> cannot be submitted.<br/>"
+            f"{message}."
         )
-        return ""
+        return model_text
+        
 
     with gr.Column():
         gr.Markdown("## Request a model here!")

espn_ldbd/leaderboard/data.py

@@ -1,12 +1,14 @@
 import os
-import time
 import threading
 import math
 from typing import Dict, Tuple
 
 import pandas as pd
 
-from datasets import Dataset, load_dataset
+from datasets import Dataset, load_dataset, concatenate_datasets
+
+from huggingface_hub import HfApi
+from huggingface_hub.utils import HfHubHTTPError 
 
 
 class LeaderboardData:
@@ -117,26 +119,89 @@ class LeaderboardData:
 
 
 class LeaderboardDataset:
-    def __init__(self):
+    def __init__(self, min_time_submit: float = 10.0):
         repo_id = os.environ.get("ESPNET_DB", None)
         assert repo_id is not None
-
+        self._repo_id = repo_id
         self._tasks = load_dataset(repo_id, "task_db", split="train")
+        self._subtasks_db = load_dataset(repo_id, "sub_task_db", split="train")
+        self._subtasks_cache = {}
         self._new_submits = []
 
         self._stop_event = threading.Event()
         self._background_submission = None
+        self._hf_api = HfApi()
+        self._submit_lock = threading.Lock()
+        self._submit_timer = None
+        self._submit_period = min_time_submit
         return
 
     @property
     def get_tasks(self):
         return self._tasks
 
+    def get_subtasks(self, task_id: str):
+        if task_id not in self._subtasks_cache:
+            subtasks = self._subtasks_db.filter(
+                lambda sample: sample["task_id"] == task_id
+            )
+            self._subtasks_cache[task_id] = subtasks
+        return self._subtasks_cache[task_id]
+
+    def submit_repoid(self, repo_id: str) -> bool:
+        # Try to acquire lock without blocking
+        if not self._submit_lock.acquire(blocking=False):
+            return False, "Submission already in progress."
+
+        message = f"Error in submitting {repo_id}."
+        try:
+            # Cancel any existing timer
+            if self._submit_timer is not None:
+                self._submit_timer.cancel()
+
+            # Set a timer to release the lock after execution
+            def _release_lock():
+                self._submit_lock.release()
+                # print(f"[Dataset] Submit lock released after timeout")
+
+            # Validate existance of repository
+            try:
+                self._hf_api.model_info(repo_id)
+            except HfHubHTTPError as e:
+                return False, "The submitted repository does not exist."
+
+            # TODO(Fhrozen): add validation of repo_id for ESPnet format
+            self._new_submits.append({
+                "model_id": repo_id,
+                "reviewed": False,
+                "date_review": "",
+                "commit_version": "",
+                "valid_repo": True,
+                "need_review": True,
+            })
+            # Set timer to auto-release lock after 5 seconds
+            self._submit_timer = threading.Timer(self._submit_period, _release_lock)
+            self._submit_timer.start()
+
+            return True, ""
+        except Exception as e:
+            # Release lock on any error
+            self._submit_lock.release()
+            # print(f"[Dataset] Error in submit_repoid: {e}")
+            message += f" {e}"
+        return False, message
+
     def _periodic_submissions_update(self):
         if len(self._new_submits) == 0:
+            print("[Dataset] No additional submits were found.")
             return
+
+        request_ds = load_dataset(self._repo_id, "request_db", split="train")
+        num_rows = len(self._new_submits)
         new_ds = Dataset.from_list(self._new_submits)
-        
+        request_ds = concatenate_datasets([request_ds, new_ds])
+        print(f"[Dataset] Adding {num_rows} new rows to request db.")
+        request_ds.push_to_hub(self._repo_id, config_name="request_db")
         self._new_submits = []
         return
 

requirements.txt

@@ -6,13 +6,13 @@ gradio
 gradio[oauth]
 gradio_client
 plotly
-huggingface-hub>=0.18.0
+huggingface-hub>=1.1.0
 matplotlib
 numpy
 pandas
 python-dateutil
 tqdm
-transformers
+transformers @ git+https://github.com/huggingface/transformers.git@2a61590
 tokenizers>=0.15.0
 sentencepiece
 python-dotenv