add basis of audio leaderboard

app.py

@@ -1,192 +1,241 @@
 import gradio as gr
-from gradio_leaderboard import Leaderboard, ColumnFilter, SelectColumns
 import pandas as pd
 from apscheduler.schedulers.background import BackgroundScheduler
+from gradio_leaderboard import Leaderboard, SelectColumns
 from huggingface_hub import snapshot_download
 
 from src.about import (
     CITATION_BUTTON_LABEL,
     CITATION_BUTTON_TEXT,
-    EVALUATION_QUEUE_TEXT,
     INTRODUCTION_TEXT,
     LLM_BENCHMARKS_TEXT,
     TITLE,
 )
 from src.display.css_html_js import custom_css
+from src.display.formatting import styled_error, styled_message
 from src.display.utils import (
-    BENCHMARK_COLS,
     COLS,
-    EVAL_COLS,
-    EVAL_TYPES,
-    AutoEvalColumn,
-    ModelType,
+    SUBMISSION_COLS,
+    SUBMISSION_TYPES,
+    TeamColumn,
     fields,
-    WeightType,
-    Precision
 )
-from src.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, QUEUE_REPO, REPO_ID, RESULTS_REPO, TOKEN
-from src.populate import get_evaluation_queue_df, get_leaderboard_df
-from src.submission.submit import add_new_eval
+from src.envs import (
+    API,
+    REPO_ID,
+    SUBMISSIONS_PATH,
+    SUBMISSIONS_REPO,
+    TEAMS_PATH,
+    TEAMS_REPO,
+    TOKEN,
+)
+from src.evaluation.load_labels import load_true_labels
+from src.populate import get_leaderboard_df, get_submission_queue_df
+from src.submission.submit_csv import submit_csv
+from src.teams.register import create_team
 
 
 def restart_space():
     API.restart_space(repo_id=REPO_ID)
 
-### Space initialisation
+
 try:
-    print(EVAL_REQUESTS_PATH)
     snapshot_download(
-        repo_id=QUEUE_REPO, local_dir=EVAL_REQUESTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN
+        repo_id=TEAMS_REPO,
+        local_dir=TEAMS_PATH,
+        repo_type="dataset",
+        tqdm_class=None,
+        etag_timeout=30,
+        token=TOKEN,
     )
 except Exception:
-    restart_space()
+    print(f"Warning: Could not download teams dataset from {TEAMS_REPO}")
+
 try:
-    print(EVAL_RESULTS_PATH)
     snapshot_download(
-        repo_id=RESULTS_REPO, local_dir=EVAL_RESULTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN
+        repo_id=SUBMISSIONS_REPO,
+        local_dir=SUBMISSIONS_PATH,
+        repo_type="dataset",
+        tqdm_class=None,
+        etag_timeout=30,
+        token=TOKEN,
     )
 except Exception:
-    restart_space()
+    print(f"Warning: Could not download submissions dataset from {SUBMISSIONS_REPO}")
 
+load_true_labels()
 
-LEADERBOARD_DF = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, COLS, BENCHMARK_COLS)
+LEADERBOARD_DF = get_leaderboard_df(SUBMISSIONS_PATH, COLS)
 
 (
-    finished_eval_queue_df,
-    running_eval_queue_df,
-    pending_eval_queue_df,
-) = get_evaluation_queue_df(EVAL_REQUESTS_PATH, EVAL_COLS)
+    accepted_submissions_df,
+    rejected_submissions_df,
+    all_submissions_df,
+) = get_submission_queue_df(SUBMISSIONS_PATH, SUBMISSION_COLS)
+
 
 def init_leaderboard(dataframe):
     if dataframe is None or dataframe.empty:
-        raise ValueError("Leaderboard DataFrame is empty or None.")
+        return Leaderboard(
+            value=pd.DataFrame(columns=COLS),
+            datatype=[c.type for c in fields(TeamColumn)],
+            interactive=False,
+        )
     return Leaderboard(
         value=dataframe,
-        datatype=[c.type for c in fields(AutoEvalColumn)],
+        datatype=[c.type for c in fields(TeamColumn)],
         select_columns=SelectColumns(
-            default_selection=[c.name for c in fields(AutoEvalColumn) if c.displayed_by_default],
-            cant_deselect=[c.name for c in fields(AutoEvalColumn) if c.never_hidden],
+            default_selection=[c.name for c in fields(TeamColumn) if c.displayed_by_default],
+            cant_deselect=[c.name for c in fields(TeamColumn) if c.never_hidden],
             label="Select Columns to Display:",
         ),
-        search_columns=[AutoEvalColumn.model.name, AutoEvalColumn.license.name],
-        hide_columns=[c.name for c in fields(AutoEvalColumn) if c.hidden],
-        filter_columns=[
-            ColumnFilter(AutoEvalColumn.model_type.name, type="checkboxgroup", label="Model types"),
-            ColumnFilter(AutoEvalColumn.precision.name, type="checkboxgroup", label="Precision"),
-            ColumnFilter(
-                AutoEvalColumn.params.name,
-                type="slider",
-                min=0.01,
-                max=150,
-                label="Select the number of parameters (B)",
-            ),
-            ColumnFilter(
-                AutoEvalColumn.still_on_hub.name, type="boolean", label="Deleted/incomplete", default=True
-            ),
-        ],
-        bool_checkboxgroup_label="Hide models",
+        search_columns=[TeamColumn.team_name.name],
+        hide_columns=[c.name for c in fields(TeamColumn) if c.hidden],
+        filter_columns=[],
         interactive=False,
     )
 
 
+def register_team_ui(team_name: str, num_teammates: int):
+    try:
+        num_teammates_int = int(num_teammates)
+    except (ValueError, TypeError):
+        return styled_error("Number of teammates must be a valid integer.")
+
+    try:
+        token, team_data = create_team(team_name, num_teammates_int)
+        return styled_message(
+            f"Team '{team_name}' registered successfully!\n\n"
+            f"**IMPORTANT: Save your token now - you won't be able to see it again!**\n\n"
+            f"Your team token: `{token}`\n\n"
+            f"Use this token to submit your predictions."
+        )
+    except ValueError as e:
+        return styled_error(str(e))
+    except Exception as e:
+        return styled_error(f"Registration failed: {str(e)}")
+
+
+def submit_csv_ui(token: str, csv_file):
+    if not token or not token.strip():
+        return styled_error("Please provide your team token.")
+
+    if csv_file is None:
+        return styled_error("Please upload a CSV file.")
+
+    try:
+        with open(csv_file.name, "r") as f:
+            csv_content = f.read()
+    except Exception as e:
+        return styled_error(f"Could not read CSV file: {str(e)}")
+
+    success, message = submit_csv(token, csv_content)
+
+    if success:
+        return styled_message(message)
+    else:
+        return styled_error(message)
+
+
 demo = gr.Blocks(css=custom_css)
 with demo:
     gr.HTML(TITLE)
     gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
 
     with gr.Tabs(elem_classes="tab-buttons") as tabs:
-        with gr.TabItem("🏅 LLM Benchmark", elem_id="llm-benchmark-tab-table", id=0):
+        with gr.TabItem("🏅 Leaderboard", elem_id="leaderboard-tab", id=0):
             leaderboard = init_leaderboard(LEADERBOARD_DF)
 
-        with gr.TabItem("📝 About", elem_id="llm-benchmark-tab-table", id=2):
+        with gr.TabItem("📝 About", elem_id="about-tab", id=1):
             gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")
 
-        with gr.TabItem("🚀 Submit here! ", elem_id="llm-benchmark-tab-table", id=3):
+        with gr.TabItem("👥 Register Team", elem_id="register-tab", id=2):
             with gr.Column():
+                gr.Markdown("## Create Your Team", elem_classes="markdown-text")
+                gr.Markdown(
+                    "Register your team to participate in the hackathon. "
+                    "You will receive a token that you'll need to submit predictions.",
+                    elem_classes="markdown-text",
+                )
+
                 with gr.Row():
-                    gr.Markdown(EVALUATION_QUEUE_TEXT, elem_classes="markdown-text")
-
-                with gr.Column():
-                    with gr.Accordion(
-                        f"✅ Finished Evaluations ({len(finished_eval_queue_df)})",
-                        open=False,
-                    ):
-                        with gr.Row():
-                            finished_eval_table = gr.components.Dataframe(
-                                value=finished_eval_queue_df,
-                                headers=EVAL_COLS,
-                                datatype=EVAL_TYPES,
-                                row_count=5,
+                    with gr.Column():
+                        team_name_input = gr.Textbox(
+                            label="Team Name",
+                            placeholder="Enter your team name",
+                            interactive=True,
+                        )
+                        num_teammates_input = gr.Number(
+                            label="Number of Teammates",
+                            value=1,
+                            minimum=1,
+                            maximum=100,
+                            step=1,
+                            interactive=True,
+                        )
+                        register_button = gr.Button("Register Team", variant="primary")
+                        registration_result = gr.Markdown()
+
+                register_button.click(
+                    register_team_ui,
+                    [team_name_input, num_teammates_input],
+                    registration_result,
+                )
+
+        with gr.TabItem("🚀 Submit Predictions", elem_id="submit-tab", id=3):
+            with gr.Column():
+                gr.Markdown("## Submit Your Predictions", elem_classes="markdown-text")
+                gr.Markdown(
+                    "Upload a CSV file with your predictions. The CSV must have two columns: "
+                    "`file_name` and `prediction`. Predictions should be binary (0/1 or 'real'/'fake').",
+                    elem_classes="markdown-text",
+                )
+
+                with gr.Row():
+                    with gr.Column():
+                        token_input = gr.Textbox(
+                            label="Team Token",
+                            placeholder="Enter your team token",
+                            type="password",
+                            interactive=True,
+                        )
+                        csv_file_input = gr.File(
+                            label="CSV File",
+                            file_types=[".csv"],
+                            interactive=True,
+                        )
+                        submit_button = gr.Button("Submit CSV", variant="primary")
+                        submission_result = gr.Markdown()
+
+                submit_button.click(
+                    submit_csv_ui,
+                    [token_input, csv_file_input],
+                    submission_result,
+                )
+
+                with gr.Accordion("📊 Submission History", open=False):
+                    with gr.Tabs():
+                        with gr.TabItem("✅ Accepted Submissions"):
+                            accepted_table = gr.components.Dataframe(
+                                value=accepted_submissions_df,
+                                headers=SUBMISSION_COLS,
+                                datatype=SUBMISSION_TYPES,
+                                row_count=10,
                             )
-                    with gr.Accordion(
-                        f"🔄 Running Evaluation Queue ({len(running_eval_queue_df)})",
-                        open=False,
-                    ):
-                        with gr.Row():
-                            running_eval_table = gr.components.Dataframe(
-                                value=running_eval_queue_df,
-                                headers=EVAL_COLS,
-                                datatype=EVAL_TYPES,
-                                row_count=5,
+                        with gr.TabItem("❌ Rejected Submissions"):
+                            rejected_table = gr.components.Dataframe(
+                                value=rejected_submissions_df,
+                                headers=SUBMISSION_COLS,
+                                datatype=SUBMISSION_TYPES,
+                                row_count=10,
                             )
-
-                    with gr.Accordion(
-                        f"⏳ Pending Evaluation Queue ({len(pending_eval_queue_df)})",
-                        open=False,
-                    ):
-                        with gr.Row():
-                            pending_eval_table = gr.components.Dataframe(
-                                value=pending_eval_queue_df,
-                                headers=EVAL_COLS,
-                                datatype=EVAL_TYPES,
-                                row_count=5,
+                        with gr.TabItem("📋 All Submissions"):
+                            all_table = gr.components.Dataframe(
+                                value=all_submissions_df,
+                                headers=SUBMISSION_COLS,
+                                datatype=SUBMISSION_TYPES,
+                                row_count=10,
                             )
-            with gr.Row():
-                gr.Markdown("# ✉️✨ Submit your model here!", elem_classes="markdown-text")
-
-            with gr.Row():
-                with gr.Column():
-                    model_name_textbox = gr.Textbox(label="Model name")
-                    revision_name_textbox = gr.Textbox(label="Revision commit", placeholder="main")
-                    model_type = gr.Dropdown(
-                        choices=[t.to_str(" : ") for t in ModelType if t != ModelType.Unknown],
-                        label="Model type",
-                        multiselect=False,
-                        value=None,
-                        interactive=True,
-                    )
-
-                with gr.Column():
-                    precision = gr.Dropdown(
-                        choices=[i.value.name for i in Precision if i != Precision.Unknown],
-                        label="Precision",
-                        multiselect=False,
-                        value="float16",
-                        interactive=True,
-                    )
-                    weight_type = gr.Dropdown(
-                        choices=[i.value.name for i in WeightType],
-                        label="Weights type",
-                        multiselect=False,
-                        value="Original",
-                        interactive=True,
-                    )
-                    base_model_name_textbox = gr.Textbox(label="Base model (for delta or adapter weights)")
-
-            submit_button = gr.Button("Submit Eval")
-            submission_result = gr.Markdown()
-            submit_button.click(
-                add_new_eval,
-                [
-                    model_name_textbox,
-                    base_model_name_textbox,
-                    revision_name_textbox,
-                    precision,
-                    weight_type,
-                    model_type,
-                ],
-                submission_result,
-            )
 
     with gr.Row():
         with gr.Accordion("📙 Citation", open=False):
@@ -201,4 +250,4 @@ with demo:
 scheduler = BackgroundScheduler()
 scheduler.add_job(restart_space, "interval", seconds=1800)
 scheduler.start()
-demo.queue(default_concurrency_limit=40).launch()
+demo.queue(default_concurrency_limit=40).launch()

requirements.txt

@@ -10,6 +10,7 @@ matplotlib
 numpy
 pandas
 python-dateutil
+scikit-learn
 tqdm
 transformers
 tokenizers>=0.15.0

src/about.py

@@ -1,70 +1,76 @@
-from dataclasses import dataclass
-from enum import Enum
+TITLE = """<h1 align="center" id="space-title">Truth vs. Machine Hackathon Leaderboard</h1>"""
 
-@dataclass
-class Task:
-    benchmark: str
-    metric: str
-    col_name: str
-
-
-# Select your tasks here
-# ---------------------------------------------------
-class Tasks(Enum):
-    # task_key in the json file, metric_key in the json file, name to display in the leaderboard 
-    task0 = Task("anli_r1", "acc", "ANLI")
-    task1 = Task("logiqa", "acc_norm", "LogiQA")
-
-NUM_FEWSHOT = 0 # Change with your few shot
-# ---------------------------------------------------
-
-
-
-# Your leaderboard name
-TITLE = """<h1 align="center" id="space-title">Demo leaderboard</h1>"""
-
-# What does your leaderboard evaluate?
 INTRODUCTION_TEXT = """
-Intro text
+Welcome to the Truth vs. Machine Hackathon Leaderboard! This leaderboard tracks teams competing in an audio deepfake detection challenge. Teams submit predictions on audio samples to determine whether they are real or fake, and the leaderboard displays the best performance metrics for each team.
 """
 
-# Which evaluations are you running? how can people reproduce what you have?
-LLM_BENCHMARKS_TEXT = f"""
+LLM_BENCHMARKS_TEXT = """
 ## How it works
 
-## Reproducibility
-To reproduce our results, here is the commands you can run:
+### 1. Register Your Team
+- Go to the "Register Team" tab
+- Enter your team name and number of teammates
+- **Save your token immediately** - you'll need it to submit predictions
+- You won't be able to see your token again after registration
+
+### 2. Prepare Your Predictions
+Create a CSV file with two columns:
+- `file_name`: The name of the audio file (must match the test set)
+- `prediction`: Your prediction (binary: 0/1, or "real"/"fake")
+
+Example CSV format:
+```csv
+file_name,prediction
+audio_001.wav,0
+audio_002.wav,1
+audio_003.wav,real
+audio_004.wav,fake
+```
 
+### 3. Submit Your Predictions
+- Go to the "Submit Predictions" tab
+- Enter your team token
+- Upload your CSV file
+- Your submission will be automatically evaluated
+
+### 4. Evaluation Metrics
+Your predictions are evaluated on:
+- **Accuracy**: Percentage of correct predictions
+- **F1 Score**: Harmonic mean of precision and recall
+- **Error Rate**: Percentage of incorrect predictions
+
+### 5. Leaderboard Updates
+- Only your **best** scores are displayed on the leaderboard
+- A submission is accepted only if it improves at least one metric
+- The leaderboard is sorted by best accuracy (primary metric)
+- If accuracy is tied, F1 score is used as a tiebreaker
+
+## Important Notes
+- True labels are kept private and not accessible to participants
+- You can submit multiple times - only your best scores count
+- Make sure your CSV file format is correct before submitting
+- File names in your CSV must exactly match the test set file names
 """
 
 EVALUATION_QUEUE_TEXT = """
-## Some good practices before submitting a model
-
-### 1) Make sure you can load your model and tokenizer using AutoClasses:
-```python
-from transformers import AutoConfig, AutoModel, AutoTokenizer
-config = AutoConfig.from_pretrained("your model name", revision=revision)
-model = AutoModel.from_pretrained("your model name", revision=revision)
-tokenizer = AutoTokenizer.from_pretrained("your model name", revision=revision)
-```
-If this step fails, follow the error messages to debug your model before submitting it. It's likely your model has been improperly uploaded.
-
-Note: make sure your model is public!
-Note: if your model needs `use_remote_code=True`, we do not support this option yet but we are working on adding it, stay posted!
-
-### 2) Convert your model weights to [safetensors](https://huggingface.co/docs/safetensors/index)
-It's a new format for storing weights which is safer and faster to load and use. It will also allow us to add the number of parameters of your model to the `Extended Viewer`!
-
-### 3) Make sure your model has an open license!
-This is a leaderboard for Open LLMs, and we'd love for as many people as possible to know they can use your model 🤗
-
-### 4) Fill up your model card
-When we add extra information about models to the leaderboard, it will be automatically taken from the model card
-
-## In case of model failure
-If your model is displayed in the `FAILED` category, its execution stopped.
-Make sure you have followed the above steps first.
-If everything is done, check you can launch the EleutherAIHarness on your model locally, using the above command without modifications (you can add `--limit` to limit the number of examples per task).
+## Submission Guidelines
+
+### CSV File Requirements
+- Must contain exactly two columns: `file_name` and `prediction`
+- `file_name` must match the test set file names exactly
+- `prediction` must be binary: 0/1 or "real"/"fake"
+- No missing values allowed
+
+### Prediction Format
+Accepted formats for predictions:
+- Numeric: `0` (real) or `1` (fake)
+- String: `"real"` or `"fake"` (case-insensitive)
+
+### Scoring
+- Submissions are evaluated immediately upon upload
+- Scores are computed using accuracy, F1 score, and error rate
+- Only submissions that improve your best scores are accepted
+- Rejected submissions are logged but don't update the leaderboard
 """
 
 CITATION_BUTTON_LABEL = "Copy the following snippet to cite these results"

src/display/utils.py

@@ -1,17 +1,10 @@
-from dataclasses import dataclass, make_dataclass
-from enum import Enum
+from dataclasses import dataclass
 
-import pandas as pd
-
-from src.about import Tasks
 
 def fields(raw_class):
     return [v for k, v in raw_class.__dict__.items() if k[:2] != "__" and k[-2:] != "__"]
 
 
-# These classes are for user facing column names,
-# to avoid having to change them all around the code
-# when a modif is needed
 @dataclass
 class ColumnContent:
     name: str
@@ -20,91 +13,27 @@ class ColumnContent:
     hidden: bool = False
     never_hidden: bool = False
 
-## Leaderboard columns
-auto_eval_column_dict = []
-# Init
-auto_eval_column_dict.append(["model_type_symbol", ColumnContent, ColumnContent("T", "str", True, never_hidden=True)])
-auto_eval_column_dict.append(["model", ColumnContent, ColumnContent("Model", "markdown", True, never_hidden=True)])
-#Scores
-auto_eval_column_dict.append(["average", ColumnContent, ColumnContent("Average ⬆️", "number", True)])
-for task in Tasks:
-    auto_eval_column_dict.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "number", True)])
-# Model information
-auto_eval_column_dict.append(["model_type", ColumnContent, ColumnContent("Type", "str", False)])
-auto_eval_column_dict.append(["architecture", ColumnContent, ColumnContent("Architecture", "str", False)])
-auto_eval_column_dict.append(["weight_type", ColumnContent, ColumnContent("Weight type", "str", False, True)])
-auto_eval_column_dict.append(["precision", ColumnContent, ColumnContent("Precision", "str", False)])
-auto_eval_column_dict.append(["license", ColumnContent, ColumnContent("Hub License", "str", False)])
-auto_eval_column_dict.append(["params", ColumnContent, ColumnContent("#Params (B)", "number", False)])
-auto_eval_column_dict.append(["likes", ColumnContent, ColumnContent("Hub ❤️", "number", False)])
-auto_eval_column_dict.append(["still_on_hub", ColumnContent, ColumnContent("Available on the hub", "bool", False)])
-auto_eval_column_dict.append(["revision", ColumnContent, ColumnContent("Model sha", "str", False, False)])
-
-# We use make dataclass to dynamically fill the scores from Tasks
-AutoEvalColumn = make_dataclass("AutoEvalColumn", auto_eval_column_dict, frozen=True)
 
-## For the queue columns in the submission tab
 @dataclass(frozen=True)
-class EvalQueueColumn:  # Queue column
-    model = ColumnContent("model", "markdown", True)
-    revision = ColumnContent("revision", "str", True)
-    private = ColumnContent("private", "bool", True)
-    precision = ColumnContent("precision", "str", True)
-    weight_type = ColumnContent("weight_type", "str", "Original")
-    status = ColumnContent("status", "str", True)
-
-## All the model information that we might need
-@dataclass
-class ModelDetails:
-    name: str
-    display_name: str = ""
-    symbol: str = "" # emoji
-
+class TeamColumn:
+    team_name = ColumnContent("Team Name", "str", True, never_hidden=True)
+    best_accuracy = ColumnContent("Best Accuracy ⬆️", "number", True)
+    best_f1 = ColumnContent("Best F1 Score", "number", True)
+    best_error_rate = ColumnContent("Best Error Rate", "number", True)
+    last_submission_date = ColumnContent("Last Submission", "str", True)
 
-class ModelType(Enum):
-    PT = ModelDetails(name="pretrained", symbol="🟢")
-    FT = ModelDetails(name="fine-tuned", symbol="🔶")
-    IFT = ModelDetails(name="instruction-tuned", symbol="⭕")
-    RL = ModelDetails(name="RL-tuned", symbol="🟦")
-    Unknown = ModelDetails(name="", symbol="?")
 
-    def to_str(self, separator=" "):
-        return f"{self.value.symbol}{separator}{self.value.name}"
-
-    @staticmethod
-    def from_str(type):
-        if "fine-tuned" in type or "🔶" in type:
-            return ModelType.FT
-        if "pretrained" in type or "🟢" in type:
-            return ModelType.PT
-        if "RL-tuned" in type or "🟦" in type:
-            return ModelType.RL
-        if "instruction-tuned" in type or "⭕" in type:
-            return ModelType.IFT
-        return ModelType.Unknown
-
-class WeightType(Enum):
-    Adapter = ModelDetails("Adapter")
-    Original = ModelDetails("Original")
-    Delta = ModelDetails("Delta")
-
-class Precision(Enum):
-    float16 = ModelDetails("float16")
-    bfloat16 = ModelDetails("bfloat16")
-    Unknown = ModelDetails("?")
-
-    def from_str(precision):
-        if precision in ["torch.float16", "float16"]:
-            return Precision.float16
-        if precision in ["torch.bfloat16", "bfloat16"]:
-            return Precision.bfloat16
-        return Precision.Unknown
-
-# Column selection
-COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden]
+@dataclass(frozen=True)
+class SubmissionQueueColumn:
+    team_name = ColumnContent("Team Name", "str", True)
+    submission_date = ColumnContent("Submission Date", "str", True)
+    accuracy = ColumnContent("Accuracy", "number", True)
+    f1 = ColumnContent("F1 Score", "number", True)
+    error_rate = ColumnContent("Error Rate", "number", True)
+    status = ColumnContent("Status", "str", True)
 
-EVAL_COLS = [c.name for c in fields(EvalQueueColumn)]
-EVAL_TYPES = [c.type for c in fields(EvalQueueColumn)]
 
-BENCHMARK_COLS = [t.value.col_name for t in Tasks]
+COLS = [c.name for c in fields(TeamColumn) if not c.hidden]
 
+SUBMISSION_COLS = [c.name for c in fields(SubmissionQueueColumn)]
+SUBMISSION_TYPES = [c.type for c in fields(SubmissionQueueColumn)]

src/envs.py

@@ -4,22 +4,22 @@ from huggingface_hub import HfApi
 
 # Info to change for your repository
 # ----------------------------------
-TOKEN = os.environ.get("HF_TOKEN") # A read/write token for your org
+TOKEN = os.environ.get("HF_TOKEN")  # A read/write token for your org
 
-OWNER = "demo-leaderboard-backend" # Change to your org - don't forget to create a results and request dataset, with the correct format!
+OWNER = "demo-leaderboard-backend"  # Change to your org - don't forget to create a results and request dataset, with the correct format!
 # ----------------------------------
 
-REPO_ID = f"{OWNER}/leaderboard"
-QUEUE_REPO = f"{OWNER}/requests"
-RESULTS_REPO = f"{OWNER}/results"
+REPO_ID = f"{OWNER}/Hackathon_Truth_Vs_Machine"
+TEAMS_REPO = f"{OWNER}/TVM_teams"
+SUBMISSIONS_REPO = f"{OWNER}/TVM_submissions"
+TRUE_LABELS_REPO = os.environ.get("TRUE_LABELS_REPO", f"{OWNER}/TVM_true-labels")
 
 # If you setup a cache later, just change HF_HOME
-CACHE_PATH=os.getenv("HF_HOME", ".")
+CACHE_PATH = os.getenv("HF_HOME", ".")
 
 # Local caches
-EVAL_REQUESTS_PATH = os.path.join(CACHE_PATH, "eval-queue")
-EVAL_RESULTS_PATH = os.path.join(CACHE_PATH, "eval-results")
-EVAL_REQUESTS_PATH_BACKEND = os.path.join(CACHE_PATH, "eval-queue-bk")
-EVAL_RESULTS_PATH_BACKEND = os.path.join(CACHE_PATH, "eval-results-bk")
+TEAMS_PATH = os.path.join(CACHE_PATH, "teams")
+SUBMISSIONS_PATH = os.path.join(CACHE_PATH, "submissions")
+TRUE_LABELS_PATH = os.path.join(CACHE_PATH, "true-labels")
 
 API = HfApi(token=TOKEN)

src/evaluation/compute_metrics.py

@@ -0,0 +1,35 @@
+import pandas as pd
+from sklearn.metrics import accuracy_score, f1_score
+
+
+def compute_metrics(predictions_df: pd.DataFrame, true_labels: dict[str, int]) -> dict[str, float]:
+    y_true = []
+    y_pred = []
+
+    for _, row in predictions_df.iterrows():
+        file_name = str(row["file_name"]).strip()
+        if file_name not in true_labels:
+            continue
+
+        true_label = true_labels[file_name]
+        pred_label = int(row["prediction"])
+
+        y_true.append(true_label)
+        y_pred.append(pred_label)
+
+    if len(y_true) == 0:
+        return {
+            "accuracy": 0.0,
+            "f1": 0.0,
+            "error_rate": 1.0,
+        }
+
+    accuracy = accuracy_score(y_true, y_pred)
+    f1 = f1_score(y_true, y_pred, zero_division=0.0)
+    error_rate = 1.0 - accuracy
+
+    return {
+        "accuracy": float(accuracy),
+        "f1": float(f1),
+        "error_rate": float(error_rate),
+    }

src/evaluation/load_labels.py

@@ -0,0 +1,53 @@
+import json
+import os
+
+from huggingface_hub import snapshot_download
+
+from src.envs import TOKEN, TRUE_LABELS_PATH, TRUE_LABELS_REPO
+
+
+def load_true_labels() -> dict[str, int]:
+    os.makedirs(TRUE_LABELS_PATH, exist_ok=True)
+
+    try:
+        snapshot_download(
+            repo_id=TRUE_LABELS_REPO,
+            local_dir=TRUE_LABELS_PATH,
+            repo_type="dataset",
+            tqdm_class=None,
+            etag_timeout=30,
+            token=TOKEN,
+        )
+    except Exception as e:
+        print(f"Warning: Could not download true labels: {e}")
+        return {}
+
+    labels = {}
+
+    for root, _, files in os.walk(TRUE_LABELS_PATH):
+        for file in files:
+            if file.endswith(".json"):
+                filepath = os.path.join(root, file)
+                try:
+                    with open(filepath, "r") as f:
+                        data = json.load(f)
+                        if isinstance(data, dict):
+                            labels.update(data)
+                        elif isinstance(data, list):
+                            for item in data:
+                                if isinstance(item, dict) and "file_name" in item and "label" in item:
+                                    labels[item["file_name"]] = item["label"]
+                except Exception:
+                    continue
+            elif file.endswith(".csv"):
+                import pandas as pd
+
+                try:
+                    df = pd.read_csv(os.path.join(root, file))
+                    if "file_name" in df.columns and "label" in df.columns:
+                        for _, row in df.iterrows():
+                            labels[str(row["file_name"])] = int(row["label"])
+                except Exception:
+                    continue
+
+    return labels

src/leaderboard/read_team_results.py

@@ -0,0 +1,52 @@
+import json
+import os
+from dataclasses import dataclass
+
+from src.display.utils import TeamColumn
+
+
+@dataclass
+class TeamResult:
+    team_name: str
+    best_accuracy: float
+    best_f1: float
+    best_error_rate: float
+    last_submission_date: str
+
+    def to_dict(self):
+        return {
+            TeamColumn.team_name.name: self.team_name,
+            TeamColumn.best_accuracy.name: self.best_accuracy,
+            TeamColumn.best_f1.name: self.best_f1,
+            TeamColumn.best_error_rate.name: self.best_error_rate,
+            TeamColumn.last_submission_date.name: self.last_submission_date,
+        }
+
+
+def get_team_results(results_path: str) -> list[TeamResult]:
+    results = []
+    results_dir = os.path.join(results_path, "results")
+
+    if not os.path.exists(results_dir):
+        return results
+
+    for filename in os.listdir(results_dir):
+        if not filename.endswith(".json"):
+            continue
+
+        filepath = os.path.join(results_dir, filename)
+        try:
+            with open(filepath, "r") as f:
+                data = json.load(f)
+                result = TeamResult(
+                    team_name=data.get("team_name", ""),
+                    best_accuracy=data.get("best_accuracy", 0.0),
+                    best_f1=data.get("best_f1", 0.0),
+                    best_error_rate=data.get("best_error_rate", 1.0),
+                    last_submission_date=data.get("last_submission_date", ""),
+                )
+                results.append(result)
+        except Exception:
+            continue
+
+    return results

src/populate.py

@@ -3,56 +3,65 @@ import os
 
 import pandas as pd
 
-from src.display.formatting import has_no_nan_values, make_clickable_model
-from src.display.utils import AutoEvalColumn, EvalQueueColumn
-from src.leaderboard.read_evals import get_raw_eval_results
+from src.display.utils import SubmissionQueueColumn, TeamColumn
+from src.leaderboard.read_team_results import get_team_results
 
 
-def get_leaderboard_df(results_path: str, requests_path: str, cols: list, benchmark_cols: list) -> pd.DataFrame:
-    """Creates a dataframe from all the individual experiment results"""
-    raw_data = get_raw_eval_results(results_path, requests_path)
+def get_leaderboard_df(results_path: str, cols: list) -> pd.DataFrame:
+    raw_data = get_team_results(results_path)
     all_data_json = [v.to_dict() for v in raw_data]
 
-    df = pd.DataFrame.from_records(all_data_json)
-    df = df.sort_values(by=[AutoEvalColumn.average.name], ascending=False)
-    df = df[cols].round(decimals=2)
+    if not all_data_json:
+        return pd.DataFrame(columns=cols)
 
-    # filter out if any of the benchmarks have not been produced
-    df = df[has_no_nan_values(df, benchmark_cols)]
+    df = pd.DataFrame.from_records(all_data_json)
+    df = df.sort_values(by=[TeamColumn.best_accuracy.name], ascending=False)
+    df = df[cols].round(decimals=4)
     return df
 
 
-def get_evaluation_queue_df(save_path: str, cols: list) -> list[pd.DataFrame]:
-    """Creates the different dataframes for the evaluation queues requestes"""
-    entries = [entry for entry in os.listdir(save_path) if not entry.startswith(".")]
-    all_evals = []
-
-    for entry in entries:
-        if ".json" in entry:
-            file_path = os.path.join(save_path, entry)
-            with open(file_path) as fp:
-                data = json.load(fp)
-
-            data[EvalQueueColumn.model.name] = make_clickable_model(data["model"])
-            data[EvalQueueColumn.revision.name] = data.get("revision", "main")
-
-            all_evals.append(data)
-        elif ".md" not in entry:
-            # this is a folder
-            sub_entries = [e for e in os.listdir(f"{save_path}/{entry}") if os.path.isfile(e) and not e.startswith(".")]
-            for sub_entry in sub_entries:
-                file_path = os.path.join(save_path, entry, sub_entry)
-                with open(file_path) as fp:
-                    data = json.load(fp)
-
-                data[EvalQueueColumn.model.name] = make_clickable_model(data["model"])
-                data[EvalQueueColumn.revision.name] = data.get("revision", "main")
-                all_evals.append(data)
-
-    pending_list = [e for e in all_evals if e["status"] in ["PENDING", "RERUN"]]
-    running_list = [e for e in all_evals if e["status"] == "RUNNING"]
-    finished_list = [e for e in all_evals if e["status"].startswith("FINISHED") or e["status"] == "PENDING_NEW_EVAL"]
-    df_pending = pd.DataFrame.from_records(pending_list, columns=cols)
-    df_running = pd.DataFrame.from_records(running_list, columns=cols)
-    df_finished = pd.DataFrame.from_records(finished_list, columns=cols)
-    return df_finished[cols], df_running[cols], df_pending[cols]
+def get_submission_queue_df(save_path: str, cols: list) -> list[pd.DataFrame]:
+    all_submissions = []
+
+    if not os.path.exists(save_path):
+        empty_df = pd.DataFrame(columns=cols)
+        return empty_df, empty_df, empty_df
+
+    for filename in os.listdir(save_path):
+        if not filename.endswith(".json"):
+            continue
+        filepath = os.path.join(save_path, filename)
+        if not os.path.isfile(filepath):
+            continue
+
+        filepath = os.path.join(save_path, filename)
+        try:
+            with open(filepath, "r") as f:
+                data = json.load(f)
+
+                submission_data = {
+                    SubmissionQueueColumn.team_name.name: data.get("team_name", ""),
+                    SubmissionQueueColumn.submission_date.name: data.get("timestamp", ""),
+                    SubmissionQueueColumn.accuracy.name: data.get("scores", {}).get("accuracy", 0.0),
+                    SubmissionQueueColumn.f1.name: data.get("scores", {}).get("f1", 0.0),
+                    SubmissionQueueColumn.error_rate.name: data.get("scores", {}).get("error_rate", 1.0),
+                    SubmissionQueueColumn.status.name: data.get("status", "UNKNOWN"),
+                }
+                all_submissions.append(submission_data)
+        except Exception:
+            continue
+
+    accepted_list = [s for s in all_submissions if s[SubmissionQueueColumn.status.name] == "ACCEPTED"]
+    rejected_list = [s for s in all_submissions if s[SubmissionQueueColumn.status.name] == "REJECTED"]
+
+    df_accepted = (
+        pd.DataFrame.from_records(accepted_list, columns=cols) if accepted_list else pd.DataFrame(columns=cols)
+    )
+    df_rejected = (
+        pd.DataFrame.from_records(rejected_list, columns=cols) if rejected_list else pd.DataFrame(columns=cols)
+    )
+    df_all = (
+        pd.DataFrame.from_records(all_submissions, columns=cols) if all_submissions else pd.DataFrame(columns=cols)
+    )
+
+    return df_accepted, df_rejected, df_all

src/submission/submit.py

@@ -1,119 +0,0 @@
-import json
-import os
-from datetime import datetime, timezone
-
-from src.display.formatting import styled_error, styled_message, styled_warning
-from src.envs import API, EVAL_REQUESTS_PATH, TOKEN, QUEUE_REPO
-from src.submission.check_validity import (
-    already_submitted_models,
-    check_model_card,
-    get_model_size,
-    is_model_on_hub,
-)
-
-REQUESTED_MODELS = None
-USERS_TO_SUBMISSION_DATES = None
-
-def add_new_eval(
-    model: str,
-    base_model: str,
-    revision: str,
-    precision: str,
-    weight_type: str,
-    model_type: str,
-):
-    global REQUESTED_MODELS
-    global USERS_TO_SUBMISSION_DATES
-    if not REQUESTED_MODELS:
-        REQUESTED_MODELS, USERS_TO_SUBMISSION_DATES = already_submitted_models(EVAL_REQUESTS_PATH)
-
-    user_name = ""
-    model_path = model
-    if "/" in model:
-        user_name = model.split("/")[0]
-        model_path = model.split("/")[1]
-
-    precision = precision.split(" ")[0]
-    current_time = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
-
-    if model_type is None or model_type == "":
-        return styled_error("Please select a model type.")
-
-    # Does the model actually exist?
-    if revision == "":
-        revision = "main"
-
-    # Is the model on the hub?
-    if weight_type in ["Delta", "Adapter"]:
-        base_model_on_hub, error, _ = is_model_on_hub(model_name=base_model, revision=revision, token=TOKEN, test_tokenizer=True)
-        if not base_model_on_hub:
-            return styled_error(f'Base model "{base_model}" {error}')
-
-    if not weight_type == "Adapter":
-        model_on_hub, error, _ = is_model_on_hub(model_name=model, revision=revision, token=TOKEN, test_tokenizer=True)
-        if not model_on_hub:
-            return styled_error(f'Model "{model}" {error}')
-
-    # Is the model info correctly filled?
-    try:
-        model_info = API.model_info(repo_id=model, revision=revision)
-    except Exception:
-        return styled_error("Could not get your model information. Please fill it up properly.")
-
-    model_size = get_model_size(model_info=model_info, precision=precision)
-
-    # Were the model card and license filled?
-    try:
-        license = model_info.cardData["license"]
-    except Exception:
-        return styled_error("Please select a license for your model")
-
-    modelcard_OK, error_msg = check_model_card(model)
-    if not modelcard_OK:
-        return styled_error(error_msg)
-
-    # Seems good, creating the eval
-    print("Adding new eval")
-
-    eval_entry = {
-        "model": model,
-        "base_model": base_model,
-        "revision": revision,
-        "precision": precision,
-        "weight_type": weight_type,
-        "status": "PENDING",
-        "submitted_time": current_time,
-        "model_type": model_type,
-        "likes": model_info.likes,
-        "params": model_size,
-        "license": license,
-        "private": False,
-    }
-
-    # Check for duplicate submission
-    if f"{model}_{revision}_{precision}" in REQUESTED_MODELS:
-        return styled_warning("This model has been already submitted.")
-
-    print("Creating eval file")
-    OUT_DIR = f"{EVAL_REQUESTS_PATH}/{user_name}"
-    os.makedirs(OUT_DIR, exist_ok=True)
-    out_path = f"{OUT_DIR}/{model_path}_eval_request_False_{precision}_{weight_type}.json"
-
-    with open(out_path, "w") as f:
-        f.write(json.dumps(eval_entry))
-
-    print("Uploading eval file")
-    API.upload_file(
-        path_or_fileobj=out_path,
-        path_in_repo=out_path.split("eval-queue/")[1],
-        repo_id=QUEUE_REPO,
-        repo_type="dataset",
-        commit_message=f"Add {model} to eval queue",
-    )
-
-    # Remove the local file
-    os.remove(out_path)
-
-    return styled_message(
-        "Your request has been submitted to the evaluation queue!\nPlease wait for up to an hour for the model to show in the PENDING list."
-    )

src/submission/submit_csv.py

@@ -0,0 +1,148 @@
+import json
+import os
+import uuid
+from datetime import datetime, timezone
+
+from src.envs import API, SUBMISSIONS_PATH, SUBMISSIONS_REPO
+from src.evaluation.compute_metrics import compute_metrics
+from src.evaluation.load_labels import load_true_labels
+from src.submission.validate_csv import validate_csv
+from src.teams.auth import validate_token
+from src.teams.storage import hash_token
+
+
+def get_team_best_scores(team_name: str) -> dict | None:
+    results_file = os.path.join(SUBMISSIONS_PATH, "results", f"{team_name}.json")
+    if os.path.exists(results_file):
+        try:
+            with open(results_file, "r") as f:
+                return json.load(f)
+        except Exception:
+            pass
+    return None
+
+
+def save_team_best_scores(team_name: str, scores: dict):
+    results_dir = os.path.join(SUBMISSIONS_PATH, "results")
+    os.makedirs(results_dir, exist_ok=True)
+    results_file = os.path.join(results_dir, f"{team_name}.json")
+
+    with open(results_file, "w") as f:
+        json.dump(scores, f)
+
+    try:
+        API.upload_file(
+            path_or_fileobj=results_file,
+            path_in_repo=f"results/{team_name}.json",
+            repo_id=SUBMISSIONS_REPO,
+            repo_type="dataset",
+            commit_message=f"Update scores for team: {team_name}",
+        )
+    except Exception as e:
+        print(f"Warning: Could not upload results to hub: {e}")
+
+
+def save_submission(team_name: str, token_hash: str, csv_content: str, scores: dict, status: str):
+    os.makedirs(SUBMISSIONS_PATH, exist_ok=True)
+    submission_id = str(uuid.uuid4())
+    timestamp = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
+
+    submission_data = {
+        "submission_id": submission_id,
+        "team_name": team_name,
+        "token_hash": token_hash,
+        "timestamp": timestamp,
+        "scores": scores,
+        "status": status,
+    }
+
+    submission_file = os.path.join(SUBMISSIONS_PATH, f"{submission_id}.json")
+    with open(submission_file, "w") as f:
+        json.dump(submission_data, f)
+
+    csv_file = os.path.join(SUBMISSIONS_PATH, f"{submission_id}.csv")
+    with open(csv_file, "w") as f:
+        f.write(csv_content)
+
+    try:
+        API.upload_file(
+            path_or_fileobj=submission_file,
+            path_in_repo=f"{submission_id}.json",
+            repo_id=SUBMISSIONS_REPO,
+            repo_type="dataset",
+            commit_message=f"Submission from {team_name}",
+        )
+        API.upload_file(
+            path_or_fileobj=csv_file,
+            path_in_repo=f"{submission_id}.csv",
+            repo_id=SUBMISSIONS_REPO,
+            repo_type="dataset",
+            commit_message=f"CSV for submission {submission_id}",
+        )
+    except Exception as e:
+        print(f"Warning: Could not upload submission to hub: {e}")
+
+
+def should_update_scores(new_scores: dict, best_scores: dict | None) -> bool:
+    if best_scores is None:
+        return True
+
+    new_accuracy = new_scores.get("accuracy", 0.0)
+    new_f1 = new_scores.get("f1", 0.0)
+    new_error = new_scores.get("error_rate", 1.0)
+
+    best_accuracy = best_scores.get("best_accuracy", 0.0)
+    best_f1 = best_scores.get("best_f1", 0.0)
+    best_error = best_scores.get("best_error_rate", 1.0)
+
+    if new_accuracy > best_accuracy:
+        return True
+    if new_accuracy == best_accuracy and new_f1 > best_f1:
+        return True
+    if new_accuracy == best_accuracy and new_f1 == best_f1 and new_error < best_error:
+        return True
+
+    return False
+
+
+def submit_csv(token: str, csv_content: str) -> tuple[bool, str]:
+    team = validate_token(token)
+    if not team:
+        return False, "Invalid token. Please check your team token."
+
+    team_name = team["team_name"]
+    token_hash = hash_token(token)
+
+    true_labels = load_true_labels()
+    if not true_labels:
+        return False, "Error: True labels not available. Please contact administrators."
+
+    is_valid, error_msg, predictions_df = validate_csv(csv_content, true_labels)
+    if not is_valid:
+        return False, f"CSV validation failed: {error_msg}"
+
+    scores = compute_metrics(predictions_df, true_labels)
+
+    best_scores = get_team_best_scores(team_name)
+
+    if should_update_scores(scores, best_scores):
+        timestamp = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
+        updated_scores = {
+            "team_name": team_name,
+            "best_accuracy": scores["accuracy"],
+            "best_f1": scores["f1"],
+            "best_error_rate": scores["error_rate"],
+            "last_submission_date": timestamp,
+        }
+        save_team_best_scores(team_name, updated_scores)
+        status = "ACCEPTED"
+        message = f"Submission accepted! Your scores: Accuracy={scores['accuracy']:.4f}, F1={scores['f1']:.4f}, Error Rate={scores['error_rate']:.4f}"
+    else:
+        status = "REJECTED"
+        best_acc = best_scores.get("best_accuracy", 0.0) if best_scores else 0.0
+        best_f1 = best_scores.get("best_f1", 0.0) if best_scores else 0.0
+        message = f"Submission rejected. Your scores (Accuracy={scores['accuracy']:.4f}, F1={scores['f1']:.4f}) did not improve your best scores (Accuracy={best_acc:.4f}, F1={best_f1:.4f})."
+
+    save_submission(team_name, token_hash, csv_content, scores, status)
+
+    return True, message

src/submission/validate_csv.py

@@ -0,0 +1,82 @@
+from io import StringIO
+
+import pandas as pd
+
+
+def normalize_prediction(pred: any) -> int | None:
+    if pd.isna(pred):
+        return None
+
+    if isinstance(pred, (int, float)):
+        if pred == 0 or pred == 1:
+            return int(pred)
+        if pred == 0.0 or pred == 1.0:
+            return int(pred)
+        return None
+
+    if isinstance(pred, str):
+        pred_lower = pred.strip().lower()
+        if pred_lower in ["0", "1", "real", "fake"]:
+            if pred_lower in ["0", "real"]:
+                return 0
+            else:
+                return 1
+        return None
+
+    return None
+
+
+def validate_csv(csv_content: str, true_labels: dict[str, int]) -> tuple[bool, str, pd.DataFrame | None]:
+    if not csv_content or not csv_content.strip():
+        return False, "CSV content is empty", None
+
+    try:
+        df = pd.read_csv(StringIO(csv_content))
+    except Exception as e:
+        return False, f"Invalid CSV format: {str(e)}", None
+
+    if "file_name" not in df.columns:
+        return False, "CSV must contain 'file_name' column", None
+
+    if "prediction" not in df.columns:
+        return False, "CSV must contain 'prediction' column", None
+
+    if df.empty:
+        return False, "CSV is empty", None
+
+    if df["file_name"].isna().any():
+        return False, "file_name column contains missing values", None
+
+    if df["prediction"].isna().any():
+        return False, "prediction column contains missing values", None
+
+    normalized_predictions = []
+    invalid_predictions = []
+
+    for idx, row in df.iterrows():
+        file_name = str(row["file_name"]).strip()
+        pred = normalize_prediction(row["prediction"])
+
+        if pred is None:
+            invalid_predictions.append(f"Row {idx + 1}: invalid prediction value '{row['prediction']}'")
+        else:
+            normalized_predictions.append(pred)
+
+    if invalid_predictions:
+        return False, "Invalid predictions found:\n" + "\n".join(invalid_predictions[:5]), None
+
+    df["prediction"] = normalized_predictions
+
+    missing_files = []
+    for file_name in df["file_name"]:
+        if str(file_name) not in true_labels:
+            missing_files.append(str(file_name))
+
+    if missing_files:
+        return (
+            False,
+            f"Unknown file names found: {', '.join(missing_files[:5])}{'...' if len(missing_files) > 5 else ''}",
+            None,
+        )
+
+    return True, "CSV is valid", df

src/teams/auth.py

@@ -0,0 +1,7 @@
+from src.teams.storage import get_team_by_token
+
+
+def validate_token(token: str) -> dict | None:
+    if not token or not token.strip():
+        return None
+    return get_team_by_token(token.strip())

src/teams/register.py

@@ -0,0 +1,19 @@
+import secrets
+
+from src.teams.storage import register_team
+
+
+def create_team(team_name: str, num_teammates: int) -> tuple[str, dict]:
+    if not team_name or not team_name.strip():
+        raise ValueError("Team name cannot be empty")
+
+    if not isinstance(num_teammates, int) or num_teammates < 1:
+        raise ValueError("Number of teammates must be a positive integer")
+
+    token = secrets.token_urlsafe(32)
+
+    try:
+        team_data = register_team(team_name.strip(), num_teammates, token)
+        return token, team_data
+    except ValueError as e:
+        raise ValueError(f"Team registration failed: {e}")

src/teams/storage.py

@@ -0,0 +1,86 @@
+import hashlib
+import json
+import os
+from datetime import datetime, timezone
+
+from src.envs import API, TEAMS_PATH, TEAMS_REPO
+
+
+def hash_token(token: str) -> str:
+    return hashlib.sha256(token.encode()).hexdigest()
+
+
+def ensure_teams_dir():
+    os.makedirs(TEAMS_PATH, exist_ok=True)
+
+
+def register_team(team_name: str, num_teammates: int, token: str) -> dict:
+    ensure_teams_dir()
+
+    token_hash = hash_token(token)
+    team_data = {
+        "team_name": team_name,
+        "num_teammates": num_teammates,
+        "token_hash": token_hash,
+        "created_at": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
+    }
+
+    team_file = os.path.join(TEAMS_PATH, f"{team_name}.json")
+
+    if os.path.exists(team_file):
+        with open(team_file, "r") as f:
+            existing = json.load(f)
+        if existing.get("token_hash") != token_hash:
+            raise ValueError("Team name already exists with different token")
+        return existing
+
+    with open(team_file, "w") as f:
+        json.dump(team_data, f)
+
+    try:
+        API.upload_file(
+            path_or_fileobj=team_file,
+            path_in_repo=f"{team_name}.json",
+            repo_id=TEAMS_REPO,
+            repo_type="dataset",
+            commit_message=f"Register team: {team_name}",
+        )
+    except Exception as e:
+        print(f"Warning: Could not upload team registration to hub: {e}")
+
+    return team_data
+
+
+def get_team_by_token(token: str) -> dict | None:
+    ensure_teams_dir()
+    token_hash = hash_token(token)
+
+    for filename in os.listdir(TEAMS_PATH):
+        if not filename.endswith(".json"):
+            continue
+        filepath = os.path.join(TEAMS_PATH, filename)
+        try:
+            with open(filepath, "r") as f:
+                team_data = json.load(f)
+                if team_data.get("token_hash") == token_hash:
+                    return team_data
+        except Exception:
+            continue
+    return None
+
+
+def get_all_teams() -> list[dict]:
+    ensure_teams_dir()
+    teams = []
+
+    for filename in os.listdir(TEAMS_PATH):
+        if not filename.endswith(".json"):
+            continue
+        filepath = os.path.join(TEAMS_PATH, filename)
+        try:
+            with open(filepath, "r") as f:
+                teams.append(json.load(f))
+        except Exception:
+            continue
+
+    return teams