fix submission check logic

src/submission/check_validity.py

@@ -103,25 +103,35 @@ def get_model_arch(model_info: ModelInfo):
 
 
 def user_submission_permission(org_or_user, users_to_submission_dates, rate_limit_period, rate_limit_quota):
+    # Increase quota first if user has higher limits
+    if org_or_user in HAS_HIGHER_RATE_LIMIT:
+        rate_limit_quota *= 2
+
     if org_or_user not in users_to_submission_dates:
         return True, ""
+
     submission_dates = sorted(users_to_submission_dates[org_or_user])
+    time_limit = datetime.now(timezone.utc) - timedelta(days=rate_limit_period)
+
+    # Convert ISO 8601 dates to datetime objects for comparison
+    def parse_datetime(date_str):
+        if date_str.endswith('Z'):
+            date_str = date_str[:-1] + '+00:00'
+        return datetime.fromisoformat(date_str)
 
-    time_limit = (datetime.now(timezone.utc) - timedelta(days=rate_limit_period)).strftime("%Y-%m-%dT%H:%M:%SZ")
-    submissions_after_timelimit = [d for d in submission_dates if d > time_limit]
+    submissions_after_timelimit = [
+        parse_datetime(d) for d in submission_dates
+        if parse_datetime(d) > time_limit
+    ]
 
     num_models_submitted_in_period = len(submissions_after_timelimit)
-    if org_or_user in HAS_HIGHER_RATE_LIMIT:
-        rate_limit_quota = 2 * rate_limit_quota
-
-    if num_models_submitted_in_period > rate_limit_quota:
-        error_msg = f"Organisation or user `{org_or_user}`"
-        error_msg += f"already has {num_models_submitted_in_period} model requests submitted to the leaderboard "
-        error_msg += f"in the last {rate_limit_period} days.\n"
-        error_msg += (
-            "Please wait a couple of days before resubmitting, so that everybody can enjoy using the leaderboard 🤗"
-        )
+
+    # Use >= to correctly enforce the rate limit
+    if num_models_submitted_in_period >= rate_limit_quota:
+        error_msg = f"Organisation or user `{org_or_user}` already has {num_models_submitted_in_period} model requests submitted in the last {rate_limit_period} days.\n"
+        error_msg += "Please wait a couple of days before resubmitting, so that everybody can enjoy using the leaderboard 🤗"
         return False, error_msg
+
     return True, ""
 
 

src/submission/submit.py

@@ -63,11 +63,6 @@ def add_new_eval(
     if model in DO_NOT_SUBMIT_MODELS or base_model in DO_NOT_SUBMIT_MODELS:
         return styled_warning("Model authors have requested that their model be not submitted on the leaderboard.")
 
-    if model == "CohereForAI/c4ai-command-r-plus":
-        return styled_warning(
-            "This model cannot be submitted manually on the leaderboard before the transformers release."
-        )
-
     # Does the model actually exist?
     if revision == "":
         revision = "main"