verbose metric

app.py

@@ -15,8 +15,8 @@ logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(level
 
 lb_server = LeaderboardServer()
 
-LEADERBOARD_TYPES = ['NOTSOFAR-SC', 'NOTSOFAR-MC', 'DASR-Constrained-LM', 'DASR-Unconstrained-LM']
-MAX_SUBMISSIONS_PER_24H = 5
+LEADERBOARD_TYPES = ['NOTSOFAR-MC', 'NOTSOFAR-SC', 'DASR-Constrained-LM', 'DASR-Unconstrained-LM']
+MAX_SUBMISSIONS_PER_24H = os.environ['MAX_SUBMISSIONS_PER_24H']
 
 
 with (gr.Blocks(theme=gr.themes.Soft(text_size=text_md), css="footer {visibility: hidden}") as main):
@@ -29,23 +29,26 @@ with (gr.Blocks(theme=gr.themes.Soft(text_size=text_md), css="footer {visibility
     with gr.Row():
         # Leaderboards Tab #
         ####################
+        def populate_leaderboard(leaderboard_type):
+            leaderboard_df = lb_server.get_leaderboard(submission_type=leaderboard_type)
+            if leaderboard_df.empty:
+                return pd.DataFrame(columns=['No submissions yet'])
+            return leaderboard_df
+
+        def create_leaderboard_tab(tab_name):
+            with gr.Tab(tab_name) as leaderboard_tab:
+                leaderboard_table = gr.DataFrame(populate_leaderboard(tab_name)) if idx == 0 \
+                    else gr.DataFrame(pd.DataFrame(columns=['No submissions yet']))
+                leaderboard_tab.select(fn=populate_leaderboard, inputs=[gr.Text(tab_name, visible=False)],
+                                       outputs=[leaderboard_table])
+                leaderboard_table.change(fn=populate_leaderboard, inputs=[gr.Text(tab_name, visible=False)],
+                                         outputs=[leaderboard_table])
+
         with gr.Tab('Leaderboards') as leaderboards_tab:
             gr.Markdown(LEADERBOARD_TAB_TITLE_MARKDOWN)
             with gr.Row():
-                def populate_leaderboard(leaderboard_type):
-                    leaderboard_df = lb_server.get_leaderboard(submission_type=leaderboard_type)
-                    if leaderboard_df.empty:
-                        return pd.DataFrame(columns=['No submissions yet'])
-                    return leaderboard_df
-
                 for idx, tab_name in enumerate(LEADERBOARD_TYPES):
-                    with gr.Tab(tab_name) as leaderboard_tab:
-                        leaderboard_table = gr.DataFrame(populate_leaderboard(tab_name)) if idx == 0 else gr.DataFrame(pd.DataFrame(columns=['No submissions yet']))
-                        leaderboard_tab.select(fn=populate_leaderboard,
-                                               inputs=[gr.Text(tab_name, visible=False)],
-                                               outputs=[leaderboard_table])
-                        leaderboard_table.change(fn=populate_leaderboard, inputs=[gr.Text(tab_name, visible=False)],
-                                                 outputs=[leaderboard_table])
+                    create_leaderboard_tab(tab_name)
 
         # Submission Tab #
         ##################
@@ -137,32 +140,33 @@ with (gr.Blocks(theme=gr.themes.Soft(text_size=text_md), css="footer {visibility
                                       outputs=[my_submissions_table])
             my_submissions_token_tb = gr.Textbox(label='Token', type='password')
 
-    # Token Insertion #
-    ###################
-    with gr.Row():
-        def on_token_insert(hf_token, app_state):
-            gr.Info(f'Verifying token...')
+    def on_token_insert(hf_token, app_state):
+        gr.Info(f'Verifying token...')
+
+        submission_count = None
+        if hf_token:
             submission_count = lb_server.get_submission_count_last_24_hours(hf_token=hf_token)
-            if submission_count is None:
-                # Invalid token
-                app_state['hf_token'] = None
-                submissions_24h_str = ''
-                team_submissions_df = pd.DataFrame(columns=['Invalid Token'])
-                gr.Warning('Invalid token')
-
-            else:
-                app_state['hf_token'] = hf_token
-                submissions_24h_str = f'{submission_count}/{MAX_SUBMISSIONS_PER_24H}'
-                team_submissions_df = lb_server.get_submissions_by_hf_token(hf_token=hf_token)
-                if team_submissions_df.empty:
-                    team_submissions_df = pd.DataFrame(columns=['No submissions yet'])
-                gr.Info('Token verified!')
-
-            return app_state, team_submissions_df, submissions_24h_str
-
-        hf_token_tb.change(fn=on_token_insert, inputs=[hf_token_tb, app_state],
-                           outputs=[app_state, my_submissions_table, submissions_24h_txt])
-        my_submissions_token_tb.change(fn=on_token_insert, inputs=[my_submissions_token_tb, app_state],
-                                       outputs=[app_state, my_submissions_table, submissions_24h_txt])
+
+        if submission_count is None:
+            # Invalid token
+            app_state['hf_token'] = None
+            submissions_24h_str = ''
+            team_submissions_df = pd.DataFrame(columns=['Invalid Token'])
+            gr.Warning('Invalid token')
+
+        else:
+            app_state['hf_token'] = hf_token
+            submissions_24h_str = f'{submission_count}/{MAX_SUBMISSIONS_PER_24H}'
+            team_submissions_df = lb_server.get_submissions_by_hf_token(hf_token=hf_token)
+            if team_submissions_df.empty:
+                team_submissions_df = pd.DataFrame(columns=['No submissions yet'])
+            gr.Info('Token verified!')
+
+        return app_state, team_submissions_df, submissions_24h_str
+
+    hf_token_tb.change(fn=on_token_insert, inputs=[hf_token_tb, app_state],
+                       outputs=[app_state, my_submissions_table, submissions_24h_txt])
+    my_submissions_token_tb.change(fn=on_token_insert, inputs=[my_submissions_token_tb, app_state],
+                                   outputs=[app_state, my_submissions_table, submissions_24h_txt])
 
     main.launch()

content.py

@@ -21,7 +21,7 @@ HEADER_MARKDOWN = """
     
     Featured in both tasks, the NOTSOFAR recorded meeting dataset is leveraged as a common benchmark: 
     each geometry-agnostic MC system submitted to DASR tracks (constrained or not) will also be **automatically submitted** 
-    to the known-geometry NOTSOFAR-MC track. These entries will be marked with "DASR" to denote their origin.
+    to the known-geometry single-device NOTSOFAR-MC track. These entries will be marked with "DASR" to denote their origin.
 """
 
 
@@ -40,12 +40,14 @@ SUBMISSION_TAB_TITLE_MARKDOWN = """
     - *Submission track:* The track to submit results to
     - *Token:* Your Hugging Face token
     - *Description:* Short description of your submission (optional)
-     
+    
     **Hugging Face tokens:** To create a token, go to your profile settings > Access Tokens > New Token.
     Name the token and give it a write role, then copy the token and paste it in the field below.\n
     **Team creation:** Upon the first submission, your team name is associated with your Hugging Face user account.  
     Any token generated by your account can be used. All team members should use this specific user's token for 
     future submissions.
+    **Submission limit:** 5 submissions per team every 24 hours. Each participant should only belong to one team. 
+    Changing team names is allowed, but it is not intended to bypass the daily submission limit.
     
     New tokens can be created by the team member who initially linked the team to the token.
 """