migrate to teams

.dockerignore

@@ -4,6 +4,7 @@
 .vim/
 flagged/
 *.csv
+*.db
 
 # Byte-compiled / optimized / DLL files
 __pycache__/

.gitignore

@@ -4,6 +4,7 @@
 .vim/
 flagged/
 *.csv
+*.db
 
 # Byte-compiled / optimized / DLL files
 __pycache__/

16337e22-7815-4ebd-a6c4-7a58dc46e214/.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

16337e22-7815-4ebd-a6c4-7a58dc46e214/script.py

@@ -0,0 +1,9 @@
+import pandas as pd
+
+
+sub = []
+for i in range(10000):
+    sub.append((i, 0.5))
+
+sub = pd.DataFrame(sub, columns=["id", "pred"])
+sub.to_csv("submission.csv", index=False)

competitions/api.py

@@ -0,0 +1,109 @@
+import asyncio
+import os
+import signal
+import sqlite3
+from contextlib import asynccontextmanager
+
+import psutil
+from fastapi import FastAPI
+from loguru import logger
+
+from competitions.utils import run_evaluation
+
+
+def get_process_status(pid):
+    try:
+        process = psutil.Process(pid)
+        proc_status = process.status()
+        return proc_status
+    except psutil.NoSuchProcess:
+        logger.info(f"No process found with PID: {pid}")
+        return "Completed"
+
+
+def kill_process_by_pid(pid):
+    """Kill process by PID."""
+    os.kill(pid, signal.SIGTERM)
+
+
+class JobDB:
+    def __init__(self, db_path):
+        self.db_path = db_path
+        self.conn = sqlite3.connect(db_path)
+        self.c = self.conn.cursor()
+        self.create_jobs_table()
+
+    def create_jobs_table(self):
+        self.c.execute(
+            """CREATE TABLE IF NOT EXISTS jobs
+            (id INTEGER PRIMARY KEY, pid INTEGER)"""
+        )
+        self.conn.commit()
+
+    def add_job(self, pid):
+        sql = f"INSERT INTO jobs (pid) VALUES ({pid})"
+        self.c.execute(sql)
+        self.conn.commit()
+
+    def get_running_jobs(self):
+        self.c.execute("""SELECT pid FROM jobs""")
+        running_pids = self.c.fetchall()
+        running_pids = [pid[0] for pid in running_pids]
+        return running_pids
+
+    def delete_job(self, pid):
+        sql = f"DELETE FROM jobs WHERE pid={pid}"
+        self.c.execute(sql)
+        self.conn.commit()
+
+
+PARAMS = os.environ.get("PARAMS")
+DB = JobDB("job.db")
+
+
+class BackgroundRunner:
+    async def run_main(self):
+        while True:
+            running_jobs = DB.get_running_jobs()
+            if running_jobs:
+                for _pid in running_jobs:
+                    proc_status = get_process_status(_pid)
+                    proc_status = proc_status.strip().lower()
+                    if proc_status in ("completed", "error", "zombie"):
+                        logger.info(f"Process {_pid} is already completed. Skipping...")
+                        try:
+                            kill_process_by_pid(_pid)
+                        except Exception as e:
+                            logger.info(f"Error while killing process: {e}")
+                        DB.delete_job(_pid)
+
+            running_jobs = DB.get_running_jobs()
+            if not running_jobs:
+                logger.info("No running jobs found. Shutting down the server.")
+                os.kill(os.getpid(), signal.SIGINT)
+            await asyncio.sleep(30)
+
+
+runner = BackgroundRunner()
+
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    process_pid = run_evaluation(params=PARAMS)
+    logger.info(f"Started training with PID {process_pid}")
+    DB.add_job(process_pid)
+    asyncio.create_task(runner.run_main())
+    yield
+
+
+api = FastAPI(lifespan=lifespan)
+
+
+@api.get("/")
+async def root():
+    return "Your model is being evaluated..."
+
+
+@api.get("/health")
+async def health():
+    return "OK"

competitions/app.py

@@ -5,6 +5,7 @@ from fastapi import FastAPI, File, Form, Request, UploadFile
 from fastapi.responses import HTMLResponse, JSONResponse
 from fastapi.staticfiles import StaticFiles
 from fastapi.templating import Jinja2Templates
+from loguru import logger
 from pydantic import BaseModel
 
 from competitions.info import CompetitionInfo
@@ -83,6 +84,7 @@ async def get_leaderboard(request: Request, lb: str):
         autotrain_token=HF_TOKEN,
     )
     df = leaderboard.fetch(private=lb == "private")
+    logger.info(df)
     resp = {"response": df.to_markdown(index=False)}
     return resp
 
@@ -94,6 +96,7 @@ async def my_submissions(request: Request, user: User):
         submission_limit=COMP_INFO.submission_limit,
         competition_id=COMPETITION_ID,
         token=HF_TOKEN,
+        competition_type=COMP_INFO.competition_type,
     )
     success_subs, failed_subs = sub.my_submissions(user.user_token)
     success_subs = success_subs.to_markdown(index=False)
@@ -107,7 +110,7 @@ async def my_submissions(request: Request, user: User):
 
 @app.post("/new_submission", response_class=JSONResponse)
 async def new_submission(
-    submission_file: UploadFile = File(...),
+    submission_file: UploadFile = File(None),
     hub_model: str = Form(...),
     token: str = Form(...),
     submission_comment: str = Form(...),
@@ -117,6 +120,7 @@ async def new_submission(
         submission_limit=COMP_INFO.submission_limit,
         competition_id=COMPETITION_ID,
         token=HF_TOKEN,
+        competition_type=COMP_INFO.competition_type,
     )
     if COMP_INFO.competition_type == "generic":
         resp = sub.new_submission(token, submission_file, submission_comment)

competitions/compute_metrics.py

@@ -15,7 +15,7 @@ def compute_metrics(params):
 
     solution_df = pd.read_csv(solution_file)
 
-    submission_filename = f"submissions/{params.user_id}-{params.submission_id}.csv"
+    submission_filename = f"submissions/{params.team_id}-{params.submission_id}.csv"
     submission_file = hf_hub_download(
         repo_id=params.competition_id,
         filename=submission_filename,
@@ -47,7 +47,7 @@ def compute_metrics(params):
     else:
         _metric = getattr(metrics, params.metric)
         target_cols = [col for col in solution_df.columns if col not in [params.submission_id_col, "split"]]
-        public_score = _metric(private_solution_df[target_cols], public_submission_df[target_cols])
+        public_score = _metric(public_solution_df[target_cols], public_submission_df[target_cols])
         private_score = _metric(private_solution_df[target_cols], private_submission_df[target_cols])
 
     # scores can also be dictionaries for multiple metrics

competitions/evaluate.py

@@ -1,7 +1,8 @@
 import argparse
 import json
+import subprocess
 
-from huggingface_hub import snapshot_download
+from huggingface_hub import HfApi, snapshot_download
 from loguru import logger
 
 from competitions import utils
@@ -15,12 +16,32 @@ def parse_args():
     return parser.parse_args()
 
 
+def upload_submission_file(params, file_path):
+    logger.info("Uploading submission file")
+    pass
+
+
 def generate_submission_file(params):
+    base_user = params.competition_id.split("/")[0]
     logger.info("Downloading submission dataset")
-    snapshot_download(
-        repo_id=params.data_path,
+    submission_dir = snapshot_download(
+        repo_id=f"{base_user}/{params.submission_id}",
         local_dir=params.output_path,
         token=params.token,
+        repo_type="model",
+    )
+    # submission_dir has a script.py file
+    # start a subprocess to run the script.py
+    # the script.py will generate a submission.csv file in the submission_dir
+    # push the submission.csv file to the repo using upload_submission_file
+    logger.info("Generating submission file")
+    subprocess.run(["python", "script.py"], cwd=submission_dir)
+
+    api = HfApi(token=params.token)
+    api.upload_file(
+        path_or_fileobj=f"{submission_dir}/submission.csv",
+        path_in_repo=f"submissions/{params.team_id}-{params.submission_id}.csv",
+        repo_id=params.competition_id,
         repo_type="dataset",
     )
 
@@ -35,9 +56,9 @@ def run(params):
     if params.competition_type == "code":
         generate_submission_file(params)
 
-    public_score, private_score = compute_metrics(params)
+    evaluation = compute_metrics(params)
 
-    utils.update_submission_score(params, public_score, private_score)
+    utils.update_submission_score(params, evaluation["public_score"], evaluation["private_score"])
     utils.update_submission_status(params, "success")
     utils.pause_space(params)
 

competitions/info.py

@@ -84,10 +84,6 @@ class CompetitionInfo:
     def competition_description(self):
         return self.competition_desc
 
-    @property
-    def competition_name(self):
-        return self.config["COMPETITION_NAME"]
-
     @property
     def submission_columns(self):
         return self.config["SUBMISSION_COLUMNS"].split(",")

competitions/leaderboard.py

@@ -25,13 +25,13 @@ class Leaderboard:
     def _refresh_columns(self):
         self.private_columns = [
             "rank",
-            "name",
+            "id",
             "private_score",
             "submission_datetime",
         ]
         self.public_columns = [
             "rank",
-            "name",
+            "id",
             "public_score",
             "submission_datetime",
         ]
@@ -49,14 +49,16 @@ class Leaderboard:
         start_time = time.time()
         submissions = []
         for submission in glob.glob(os.path.join(submissions_folder, "submission_info", "*.json")):
-            with open(submission, "r") as f:
+            with open(submission, "r", encoding="utf-8") as f:
                 submission_info = json.load(f)
             # only select submissions that are done
-            submission_info["submissions"] = [sub for sub in submission_info["submissions"] if sub["status"] == "done"]
+            submission_info["submissions"] = [
+                sub for sub in submission_info["submissions"] if sub["status"] == "success"
+            ]
             submission_info["submissions"] = [
                 sub
                 for sub in submission_info["submissions"]
-                if datetime.strptime(sub["date"], "%Y-%m-%d") < self.end_date
+                if datetime.strptime(sub["datetime"], "%Y-%m-%d %H:%M:%S") < self.end_date
             ]
             if len(submission_info["submissions"]) == 0:
                 continue
@@ -83,15 +85,13 @@ class Leaderboard:
             submission_info["submissions"] = submission_info["submissions"][0]
             temp_info = {
                 "id": submission_info["id"],
-                "name": submission_info["name"],
                 "submission_id": submission_info["submissions"]["submission_id"],
                 "submission_comment": submission_info["submissions"]["submission_comment"],
                 "status": submission_info["submissions"]["status"],
                 "selected": submission_info["submissions"]["selected"],
                 "public_score": submission_info["submissions"]["public_score"],
                 # "private_score": submission_info["submissions"]["private_score"],
-                "submission_date": submission_info["submissions"]["date"],
-                "submission_time": submission_info["submissions"]["time"],
+                "submission_datetime": submission_info["submissions"]["datetime"],
             }
             for score in other_scores:
                 temp_info[score] = submission_info["submissions"][score]
@@ -112,10 +112,10 @@ class Leaderboard:
         start_time = time.time()
         submissions = []
         for submission in glob.glob(os.path.join(submissions_folder, "submission_info", "*.json")):
-            with open(submission, "r") as f:
+            with open(submission, "r", encoding="utf-8") as f:
                 submission_info = json.load(f)
                 submission_info["submissions"] = [
-                    sub for sub in submission_info["submissions"] if sub["status"] == "done"
+                    sub for sub in submission_info["submissions"] if sub["status"] == "success"
                 ]
                 if len(submission_info["submissions"]) == 0:
                     continue
@@ -146,8 +146,7 @@ class Leaderboard:
                 if selected_submissions == 0:
                     # select submissions with best public score
                     submission_info["submissions"].sort(
-                        key=lambda x: x["public_score"],
-                        reverse=True if self.eval_higher_is_better else False,
+                        key=lambda x: x["public_score"], reverse=self.eval_higher_is_better
                     )
                     # select only the best submission
                     submission_info["submissions"] = submission_info["submissions"][0]
@@ -156,8 +155,7 @@ class Leaderboard:
                     submission_info["submissions"] = [sub for sub in submission_info["submissions"] if sub["selected"]]
                     # sort by private score
                     submission_info["submissions"].sort(
-                        key=lambda x: x["private_score"],
-                        reverse=True if self.eval_higher_is_better else False,
+                        key=lambda x: x["private_score"], reverse=self.eval_higher_is_better
                     )
                     # select only the best submission
                     submission_info["submissions"] = submission_info["submissions"][0]
@@ -167,28 +165,24 @@ class Leaderboard:
                         sub for sub in submission_info["submissions"] if not sub["selected"]
                     ]
                     temp_best_public_submissions.sort(
-                        key=lambda x: x["public_score"],
-                        reverse=True if self.eval_higher_is_better else False,
+                        key=lambda x: x["public_score"], reverse=self.eval_higher_is_better
                     )
                     missing_candidates = self.max_selected_submissions - len(temp_selected_submissions)
                     temp_best_public_submissions = temp_best_public_submissions[:missing_candidates]
                     submission_info["submissions"] = temp_selected_submissions + temp_best_public_submissions
                     submission_info["submissions"].sort(
-                        key=lambda x: x["private_score"],
-                        reverse=True if self.eval_higher_is_better else False,
+                        key=lambda x: x["private_score"], reverse=self.eval_higher_is_better
                     )
                     submission_info["submissions"] = submission_info["submissions"][0]
 
                 temp_info = {
                     "id": submission_info["id"],
-                    "name": submission_info["name"],
                     "submission_id": submission_info["submissions"]["submission_id"],
                     "submission_comment": submission_info["submissions"]["submission_comment"],
                     "status": submission_info["submissions"]["status"],
                     "selected": submission_info["submissions"]["selected"],
                     "private_score": submission_info["submissions"]["private_score"],
-                    "submission_date": submission_info["submissions"]["date"],
-                    "submission_time": submission_info["submissions"]["time"],
+                    "submission_datetime": submission_info["submissions"]["datetime"],
                 }
                 for score in other_scores:
                     temp_info[score] = submission_info["submissions"][score]
@@ -206,10 +200,10 @@ class Leaderboard:
             return pd.DataFrame()
 
         df = pd.DataFrame(submissions)
-        # convert submission date and time to datetime
-        df["submission_datetime"] = pd.to_datetime(
-            df["submission_date"] + " " + df["submission_time"], format="%Y-%m-%d %H:%M:%S"
-        )
+
+        # convert submission datetime to pandas datetime
+        df["submission_datetime"] = pd.to_datetime(df["submission_datetime"], format="%Y-%m-%d %H:%M:%S")
+
         # only keep submissions before the end date
         df = df[df["submission_datetime"] < self.end_date].reset_index(drop=True)
 
@@ -254,7 +248,6 @@ class Leaderboard:
         columns = self.public_columns if not private else self.private_columns
         logger.info(columns)
         # remove duplicate columns
-        # ['rank', 'name', 'public_score', 'submission_datetime', 'public_score_track1', 'public_score_track1', 'public_score_track1', 'public_score_track1']
         columns = list(dict.fromkeys(columns))
 
         # send submission_datetime to the end

competitions/params.py

@@ -9,12 +9,13 @@ class EvalParams(BaseModel):
     competition_type: str
     metric: str
     token: str
-    user_id: str
+    team_id: str
     submission_id: str
     submission_id_col: str
     submission_cols: List[str]
     submission_rows: int
     output_path: str
+    submission_repo: str
 
     class Config:
         protected_namespaces = ()

competitions/runner.py

@@ -1,17 +1,30 @@
 import glob
+import io
 import json
 import os
+import random
+import string
 import time
 from dataclasses import dataclass
 
 import pandas as pd
-from huggingface_hub import snapshot_download
+from huggingface_hub import HfApi, snapshot_download
 from loguru import logger
 
 from competitions.info import CompetitionInfo
 from competitions.utils import run_evaluation
 
 
+_DOCKERFILE = """
+FROM huggingface/competitions:latest
+
+CMD uvicorn competitions.app:app --port 7860 --host 0.0.0.0
+"""
+
+# format _DOCKERFILE
+_DOCKERFILE = _DOCKERFILE.replace("\n", " ").replace("  ", "\n").strip()
+
+
 @dataclass
 class JobRunner:
     competition_info: CompetitionInfo
@@ -27,56 +40,121 @@ class JobRunner:
         self.submission_rows = self.competition_info.submission_rows
 
     def get_pending_subs(self):
-        user_jsons = snapshot_download(
+        submission_jsons = snapshot_download(
             repo_id=self.competition_id,
             allow_patterns="submission_info/*.json",
             token=self.token,
             repo_type="dataset",
         )
-        user_jsons = glob.glob(os.path.join(user_jsons, "submission_info/*.json"))
+        submission_jsons = glob.glob(os.path.join(submission_jsons, "submission_info/*.json"))
         pending_submissions = []
-        for _json in user_jsons:
+        for _json in submission_jsons:
             _json = json.load(open(_json, "r", encoding="utf-8"))
-            user_id = _json["id"]
+            team_id = _json["id"]
             for sub in _json["submissions"]:
-                # if sub["status"] == "pending":
-                pending_submissions.append(
-                    {
-                        "user_id": user_id,
-                        "submission_id": sub["submission_id"],
-                        "date": sub["date"],
-                        "time": sub["time"],
-                    }
-                )
+                if sub["status"] == "pending":
+                    pending_submissions.append(
+                        {
+                            "team_id": team_id,
+                            "submission_id": sub["submission_id"],
+                            "datetime": sub["datetime"],
+                            "submission_repo": sub["submission_repo"],
+                        }
+                    )
         if len(pending_submissions) == 0:
             logger.info("No pending submissions.")
             return None
         logger.info(f"Found {len(pending_submissions)} pending submissions.")
         pending_submissions = pd.DataFrame(pending_submissions)
-        pending_submissions = pending_submissions.sort_values(by=["date", "time"])
+        pending_submissions["datetime"] = pd.to_datetime(pending_submissions["datetime"])
+        pending_submissions = pending_submissions.sort_values("datetime")
         pending_submissions = pending_submissions.reset_index(drop=True)
         return pending_submissions
 
     def run_local(self, pending_submissions):
         for _, row in pending_submissions.iterrows():
-            user_id = row["user_id"]
+            team_id = row["team_id"]
             submission_id = row["submission_id"]
             eval_params = {
                 "competition_id": self.competition_id,
                 "competition_type": self.competition_type,
                 "metric": self.metric,
                 "token": self.token,
-                "user_id": user_id,
+                "team_id": team_id,
                 "submission_id": submission_id,
                 "submission_id_col": self.submission_id_col,
                 "submission_cols": self.submission_cols,
                 "submission_rows": self.submission_rows,
                 "output_path": self.output_path,
+                "submission_repo": row["submission_repo"],
             }
             eval_params = json.dumps(eval_params)
             eval_pid = run_evaluation(eval_params, local=True, wait=True)
             logger.info(f"New evaluation process started with pid {eval_pid}.")
 
+    def _create_readme(self, project_name):
+        _readme = "---\n"
+        _readme += f"title: {project_name}\n"
+        _readme += "emoji: 🚀\n"
+        _readme += "colorFrom: green\n"
+        _readme += "colorTo: indigo\n"
+        _readme += "sdk: docker\n"
+        _readme += "pinned: false\n"
+        _readme += "duplicated_from: autotrain-projects/autotrain-advanced\n"
+        _readme += "---\n"
+        _readme = io.BytesIO(_readme.encode())
+        return _readme
+
+    def create_space(self, team_id, submission_id, submission_repo):
+        project_name = "".join(
+            random.choices(
+                string.ascii_lowercase + string.digits,
+                k=10,
+            )
+        )
+        api = HfApi(token=self.token)
+        username = self.competition_id.split("/")[0]
+        repo_id = f"{username}/competitions-{project_name}"
+        api.create_repo(
+            repo_id=repo_id,
+            repo_type="space",
+            space_sdk="docker",
+            space_hardware="cpu-basic",
+            private=True,
+        )
+        params = {
+            "competition_id": self.competition_id,
+            "competition_type": self.competition_type,
+            "metric": self.metric,
+            "token": self.token,
+            "team_id": team_id,
+            "submission_id": submission_id,
+            "submission_id_col": self.submission_id_col,
+            "submission_cols": self.submission_cols,
+            "submission_rows": self.submission_rows,
+            "output_path": self.output_path,
+            "submission_repo": submission_repo,
+        }
+
+        api.add_space_secret(repo_id=repo_id, key="PARAMS", value=json.dumps(params))
+
+        readme = self._create_readme(project_name)
+        api.upload_file(
+            path_or_fileobj=readme,
+            path_in_repo="README.md",
+            repo_id=repo_id,
+            repo_type="space",
+        )
+
+        _dockerfile = io.BytesIO(_DOCKERFILE.encode())
+        api.upload_file(
+            path_or_fileobj=_dockerfile,
+            path_in_repo="Dockerfile",
+            repo_id=repo_id,
+            repo_type="space",
+        )
+        return repo_id
+
     def run(self):
         while True:
             pending_submissions = self.get_pending_subs()
@@ -85,4 +163,10 @@ class JobRunner:
                 continue
             if self.competition_type == "generic":
                 self.run_local(pending_submissions)
+            elif self.competition_type == "code":
+                for _, row in pending_submissions.iterrows():
+                    team_id = row["team_id"]
+                    submission_id = row["submission_id"]
+                    submission_repo = row["submission_repo"]
+                    self.create_space(team_id, submission_id, submission_repo)
             time.sleep(5)

competitions/submissions.py

@@ -5,7 +5,7 @@ from dataclasses import dataclass
 from datetime import datetime
 
 import pandas as pd
-from huggingface_hub import HfApi, hf_hub_download
+from huggingface_hub import HfApi, hf_hub_download, snapshot_download
 from huggingface_hub.utils._errors import EntryNotFoundError
 from loguru import logger
 
@@ -16,6 +16,7 @@ from .utils import user_authentication
 @dataclass
 class Submissions:
     competition_id: str
+    competition_type: str
     submission_limit: str
     end_date: datetime
     token: str
@@ -42,38 +43,35 @@ class Submissions:
     def _verify_submission(self, bytes_data):
         return True
 
-    def _add_new_user(self, user_info):
+    def _add_new_team(self, team_id):
         api = HfApi(token=self.token)
-        user_submission_info = {}
-        user_submission_info["name"] = user_info["name"]
-        user_submission_info["id"] = user_info["id"]
-        user_submission_info["submissions"] = []
-        # convert user_submission_info to BufferedIOBase file object
-        user_submission_info_json = json.dumps(user_submission_info, indent=4)
-        user_submission_info_json_bytes = user_submission_info_json.encode("utf-8")
-        user_submission_info_json_buffer = io.BytesIO(user_submission_info_json_bytes)
+        team_submission_info = {}
+        team_submission_info["id"] = team_id
+        team_submission_info["submissions"] = []
+        team_submission_info_json = json.dumps(team_submission_info, indent=4)
+        team_submission_info_json_bytes = team_submission_info_json.encode("utf-8")
+        team_submission_info_json_buffer = io.BytesIO(team_submission_info_json_bytes)
 
         api.upload_file(
-            path_or_fileobj=user_submission_info_json_buffer,
-            path_in_repo=f"submission_info/{user_info['id']}.json",
+            path_or_fileobj=team_submission_info_json_buffer,
+            path_in_repo=f"submission_info/{team_id}.json",
             repo_id=self.competition_id,
             repo_type="dataset",
         )
 
-    def _check_user_submission_limit(self, user_info):
-        user_id = user_info["id"]
+    def _check_team_submission_limit(self, team_id):
         try:
-            user_fname = hf_hub_download(
+            team_fname = hf_hub_download(
                 repo_id=self.competition_id,
-                filename=f"submission_info/{user_id}.json",
+                filename=f"submission_info/{team_id}.json",
                 token=self.token,
                 repo_type="dataset",
             )
         except EntryNotFoundError:
-            self._add_new_user(user_info)
-            user_fname = hf_hub_download(
+            self._add_new_team(team_id)
+            team_fname = hf_hub_download(
                 repo_id=self.competition_id,
-                filename=f"submission_info/{user_id}.json",
+                filename=f"submission_info/{team_id}.json",
                 token=self.token,
                 repo_type="dataset",
             )
@@ -81,36 +79,37 @@ class Submissions:
             logger.error(e)
             raise Exception("Hugging Face Hub is unreachable, please try again later.")
 
-        with open(user_fname, "r", encoding="utf-8") as f:
-            user_submission_info = json.load(f)
+        with open(team_fname, "r", encoding="utf-8") as f:
+            team_submission_info = json.load(f)
 
         todays_date = datetime.now().strftime("%Y-%m-%d")
-        if len(user_submission_info["submissions"]) == 0:
-            user_submission_info["submissions"] = []
+        if len(team_submission_info["submissions"]) == 0:
+            team_submission_info["submissions"] = []
 
         # count the number of times user has submitted today
         todays_submissions = 0
-        for sub in user_submission_info["submissions"]:
-            if sub["date"] == todays_date:
+        for sub in team_submission_info["submissions"]:
+            submission_datetime = sub["datetime"]
+            submission_date = submission_datetime.split(" ")[0]
+            if submission_date == todays_date:
                 todays_submissions += 1
         if todays_submissions >= self.submission_limit:
             return False
         return True
 
-    def _submissions_today(self, user_info):
-        user_id = user_info["id"]
+    def _submissions_today(self, team_id):
         try:
-            user_fname = hf_hub_download(
+            team_fname = hf_hub_download(
                 repo_id=self.competition_id,
-                filename=f"submission_info/{user_id}.json",
+                filename=f"submission_info/{team_id}.json",
                 token=self.token,
                 repo_type="dataset",
             )
         except EntryNotFoundError:
-            self._add_new_user(user_info)
-            user_fname = hf_hub_download(
+            self._add_new_team(team_id)
+            team_fname = hf_hub_download(
                 repo_id=self.competition_id,
-                filename=f"submission_info/{user_id}.json",
+                filename=f"submission_info/{team_id}.json",
                 token=self.token,
                 repo_type="dataset",
             )
@@ -118,39 +117,43 @@ class Submissions:
             logger.error(e)
             raise Exception("Hugging Face Hub is unreachable, please try again later.")
 
-        with open(user_fname, "r", encoding="utf-8") as f:
-            user_submission_info = json.load(f)
+        with open(team_fname, "r", encoding="utf-8") as f:
+            team_submission_info = json.load(f)
 
         todays_date = datetime.now().strftime("%Y-%m-%d")
-        if len(user_submission_info["submissions"]) == 0:
-            user_submission_info["submissions"] = []
+        if len(team_submission_info["submissions"]) == 0:
+            team_submission_info["submissions"] = []
 
         # count the number of times user has submitted today
         todays_submissions = 0
-        for sub in user_submission_info["submissions"]:
-            if sub["date"] == todays_date:
+        for sub in team_submission_info["submissions"]:
+            submission_datetime = sub["datetime"]
+            submission_date = submission_datetime.split(" ")[0]
+            if submission_date == todays_date:
                 todays_submissions += 1
         return todays_submissions
 
-    def _increment_submissions(self, user_id, submission_id, submission_comment):
-        user_fname = hf_hub_download(
+    def _increment_submissions(self, team_id, user_id, submission_id, submission_comment, submission_repo=None):
+        if submission_repo is None:
+            submission_repo = ""
+        team_fname = hf_hub_download(
             repo_id=self.competition_id,
-            filename=f"submission_info/{user_id}.json",
+            filename=f"submission_info/{team_id}.json",
             token=self.token,
             repo_type="dataset",
         )
-        with open(user_fname, "r", encoding="utf-8") as f:
-            user_submission_info = json.load(f)
-        todays_date = datetime.now().strftime("%Y-%m-%d")
-        current_time = datetime.now().strftime("%H:%M:%S")
+        with open(team_fname, "r", encoding="utf-8") as f:
+            team_submission_info = json.load(f)
+        datetime_now = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
 
         # here goes all the default stuff for submission
-        user_submission_info["submissions"].append(
+        team_submission_info["submissions"].append(
             {
-                "date": todays_date,
-                "time": current_time,
+                "datetime": datetime_now,
                 "submission_id": submission_id,
                 "submission_comment": submission_comment,
+                "submission_repo": submission_repo,
+                "submitted_by": user_id,
                 "status": "pending",
                 "selected": False,
                 "public_score": -1,
@@ -159,33 +162,35 @@ class Submissions:
         )
         # count the number of times user has submitted today
         todays_submissions = 0
-        for sub in user_submission_info["submissions"]:
-            if sub["date"] == todays_date:
+        todays_date = datetime.now().strftime("%Y-%m-%d")
+        for sub in team_submission_info["submissions"]:
+            submission_datetime = sub["datetime"]
+            submission_date = submission_datetime.split(" ")[0]
+            if submission_date == todays_date:
                 todays_submissions += 1
 
-        # convert user_submission_info to BufferedIOBase file object
-        user_submission_info_json = json.dumps(user_submission_info, indent=4)
-        user_submission_info_json_bytes = user_submission_info_json.encode("utf-8")
-        user_submission_info_json_buffer = io.BytesIO(user_submission_info_json_bytes)
+        team_submission_info_json = json.dumps(team_submission_info, indent=4)
+        team_submission_info_json_bytes = team_submission_info_json.encode("utf-8")
+        team_submission_info_json_buffer = io.BytesIO(team_submission_info_json_bytes)
         api = HfApi(token=self.token)
         api.upload_file(
-            path_or_fileobj=user_submission_info_json_buffer,
-            path_in_repo=f"submission_info/{user_id}.json",
+            path_or_fileobj=team_submission_info_json_buffer,
+            path_in_repo=f"submission_info/{team_id}.json",
             repo_id=self.competition_id,
             repo_type="dataset",
         )
         return todays_submissions
 
-    def _download_user_subs(self, user_id):
-        user_fname = hf_hub_download(
+    def _download_team_subs(self, team_id):
+        team_fname = hf_hub_download(
             repo_id=self.competition_id,
-            filename=f"submission_info/{user_id}.json",
+            filename=f"submission_info/{team_id}.json",
             token=self.token,
             repo_type="dataset",
         )
-        with open(user_fname, "r", encoding="utf-8") as f:
-            user_submission_info = json.load(f)
-        return user_submission_info["submissions"]
+        with open(team_fname, "r", encoding="utf-8") as f:
+            team_submission_info = json.load(f)
+        return team_submission_info["submissions"]
 
     def update_selected_submissions(self, user_token, selected_submission_ids):
         current_datetime = datetime.now()
@@ -194,44 +199,44 @@ class Submissions:
 
         user_info = self._get_user_info(user_token)
         user_id = user_info["id"]
+        team_id = self._get_team_id(user_id)
 
-        user_fname = hf_hub_download(
+        team_fname = hf_hub_download(
             repo_id=self.competition_id,
-            filename=f"submission_info/{user_id}.json",
+            filename=f"submission_info/{team_id}.json",
             token=self.token,
             repo_type="dataset",
         )
-        with open(user_fname, "r", encoding="utf-8") as f:
-            user_submission_info = json.load(f)
+        with open(team_fname, "r", encoding="utf-8") as f:
+            team_submission_info = json.load(f)
 
-        for sub in user_submission_info["submissions"]:
+        for sub in team_submission_info["submissions"]:
             if sub["submission_id"] in selected_submission_ids:
                 sub["selected"] = True
             else:
                 sub["selected"] = False
 
-        # convert user_submission_info to BufferedIOBase file object
-        user_submission_info_json = json.dumps(user_submission_info, indent=4)
-        user_submission_info_json_bytes = user_submission_info_json.encode("utf-8")
-        user_submission_info_json_buffer = io.BytesIO(user_submission_info_json_bytes)
+        team_submission_info_json = json.dumps(team_submission_info, indent=4)
+        team_submission_info_json_bytes = team_submission_info_json.encode("utf-8")
+        team_submission_info_json_buffer = io.BytesIO(team_submission_info_json_bytes)
         api = HfApi(token=self.token)
         api.upload_file(
-            path_or_fileobj=user_submission_info_json_buffer,
-            path_in_repo=f"submission_info/{user_id}.json",
+            path_or_fileobj=team_submission_info_json_buffer,
+            path_in_repo=f"submission_info/{team_id}.json",
             repo_id=self.competition_id,
             repo_type="dataset",
         )
 
-    def _get_user_subs(self, user_info, private=False):
-        # get user submissions
+    def _get_team_subs(self, user_info, private=False):
         user_id = user_info["id"]
+        team_id = self._get_team_id(user_id)
         try:
-            user_submissions = self._download_user_subs(user_id)
+            team_submissions = self._download_team_subs(team_id)
         except EntryNotFoundError:
             logger.warning("No submissions found for user")
             return pd.DataFrame(), pd.DataFrame()
 
-        submissions_df = pd.DataFrame(user_submissions)
+        submissions_df = pd.DataFrame(team_submissions)
 
         if not private:
             submissions_df = submissions_df.drop(columns=["private_score"])
@@ -314,47 +319,95 @@ class Submissions:
         private = False
         if current_date_time >= self.end_date:
             private = True
-        success_subs, failed_subs = self._get_user_subs(user_info, private=private)
+        success_subs, failed_subs = self._get_team_subs(user_info, private=private)
         return success_subs, failed_subs
 
+    def _get_team_id(self, user_id):
+        user_team = hf_hub_download(
+            repo_id=self.competition_id,
+            filename="user_team.json",
+            token=self.token,
+            repo_type="dataset",
+        )
+        with open(user_team, "r", encoding="utf-8") as f:
+            user_team = json.load(f)
+
+        if user_id in user_team:
+            return user_team[user_id]
+
+        # create a new team, if user is not in any team
+        team_id = str(uuid.uuid4())
+        user_team[user_id] = team_id
+        user_team_json = json.dumps(user_team, indent=4)
+        user_team_json_bytes = user_team_json.encode("utf-8")
+        user_team_json_buffer = io.BytesIO(user_team_json_bytes)
+        api = HfApi(token=self.token)
+        api.upload_file(
+            path_or_fileobj=user_team_json_buffer,
+            path_in_repo="user_team.json",
+            repo_id=self.competition_id,
+            repo_type="dataset",
+        )
+        return team_id
+
     def new_submission(self, user_token, uploaded_file, submission_comment):
         # verify token
         user_info = self._get_user_info(user_token)
+        submission_id = str(uuid.uuid4())
+        user_id = user_info["id"]
+        team_id = self._get_team_id(user_id)
 
-        # check if user can submit to the competition
-        if self._check_user_submission_limit(user_info) is False:
+        # check if team can submit to the competition
+        if self._check_team_submission_limit(team_id) is False:
             raise SubmissionLimitError("Submission limit reached")
 
-        logger.info(type(uploaded_file))
-        bytes_data = uploaded_file.file.read()
+        if self.competition_type == "generic":
+            bytes_data = uploaded_file.file.read()
+            # verify file is valid
+            if not self._verify_submission(bytes_data):
+                raise SubmissionError("Invalid submission file")
 
-        # verify file is valid
-        if not self._verify_submission(bytes_data):
-            raise SubmissionError("Invalid submission file")
-        else:
-            user_id = user_info["id"]
-            submission_id = str(uuid.uuid4())
             file_extension = uploaded_file.filename.split(".")[-1]
             # upload file to hf hub
             api = HfApi(token=self.token)
             api.upload_file(
                 path_or_fileobj=bytes_data,
-                path_in_repo=f"submissions/{user_id}-{submission_id}.{file_extension}",
+                path_in_repo=f"submissions/{team_id}-{submission_id}.{file_extension}",
                 repo_id=self.competition_id,
                 repo_type="dataset",
             )
-            # update submission limit
             submissions_made = self._increment_submissions(
+                team_id=team_id,
+                user_id=user_id,
+                submission_id=submission_id,
+                submission_comment=submission_comment,
+                submission_repo="",
+            )
+        else:
+            submission_repo = snapshot_download(
+                repo_id=uploaded_file,
+                local_dir=submission_id,
+                token=user_token,
+                repo_type="model",
+            )
+            api = HfApi(token=self.token)
+            competition_user = self.competition_id.split("/")[0]
+            api.create_repo(
+                repo_id=f"{competition_user}/{submission_id}",
+                repo_type="model",
+                private=True,
+            )
+            api.upload_folder(
+                folder_path=submission_repo,
+                repo_id=f"{competition_user}/{submission_id}",
+                repo_type="model",
+            )
+            submissions_made = self._increment_submissions(
+                team_id=team_id,
                 user_id=user_id,
                 submission_id=submission_id,
-                submission_comment="",
+                submission_comment=submission_comment,
+                submission_repo=uploaded_file,
             )
-            # TODO: schedule submission for evaluation
-            # self._create_autotrain_project(
-            #     submission_id=f"{submission_id}",
-            #     competition_id=f"{self.competition_id}",
-            #     user_id=user_id,
-            #     competition_type="generic",
-            # )
         remaining_submissions = self.submission_limit - submissions_made
         return remaining_submissions

competitions/utils.py

@@ -73,7 +73,7 @@ def pause_space(params):
 def download_submission_info(params):
     user_fname = hf_hub_download(
         repo_id=params.competition_id,
-        filename=f"submission_info/{params.user_id}.json",
+        filename=f"submission_info/{params.team_id}.json",
         token=params.token,
         repo_type="dataset",
     )
@@ -90,7 +90,7 @@ def upload_submission_info(params, user_submission_info):
     api = HfApi(token=params.token)
     api.upload_file(
         path_or_fileobj=user_submission_info_json_buffer,
-        path_in_repo=f"submission_info/{params.user_id}.json",
+        path_in_repo=f"submission_info/{params.team_id}.json",
         repo_id=params.competition_id,
         repo_type="dataset",
     )