submission-fix (#803)

- fix submission check logic (e6d10f6c0c8d5d5bdc367a0fbf4f2545bb8d64d1)
- add line break for doc sentence (35a2b072c7ad323cb2dc4af7446cae596006a181)
- fix tests and add new function to utils.py (e641f846e40711431d2eb3cb2163f7573f09fe35)
- update gitignore (13df6817e1929893cb4f38667951e88fd625538c)

.gitignore

@@ -10,10 +10,13 @@ __pycache__/
 .python-version
 .profile_app.python
 *pstats
-poetry.lock
+*.lock
 
 eval-queue/
 eval-results/
 dynamic-info/
+downloads/
+model-votes/
+open-llm-leaderboard___contents/
 
 src/assets/model_counts.html

src/display/about.py

@@ -4,7 +4,7 @@ TITLE = """<h1 style="text-align:left;float:left; id="space-title">🤗 Open LLM
 
 INTRODUCTION_TEXT = """
 Feeling lost? Documentation is [here](https://huggingface.co/docs/leaderboards/open_llm_leaderboard/about) 📄
-You'll notably find explanations on the evaluations we are using, reproducibility guidelines, best practices on how to submit a model, and our FAQ.
+\n You'll notably find explanations on the evaluations we are using, reproducibility guidelines, best practices on how to submit a model, and our FAQ.
 """
 
 icons = f"""

src/display/utils.py

@@ -9,6 +9,11 @@ import pandas as pd
 # Configure logging
 logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s")
 
+# Convert ISO 8601 dates to datetime objects for comparison
+def parse_iso8601_datetime(date_str):
+    if date_str.endswith('Z'):
+        date_str = date_str[:-1] + '+00:00'
+    return datetime.fromisoformat(date_str)
 
 def parse_datetime(datetime_str):
     formats = [

src/submission/check_validity.py

@@ -9,6 +9,7 @@ from huggingface_hub import ModelCard
 from huggingface_hub.hf_api import ModelInfo, get_safetensors_metadata
 from transformers import AutoConfig, AutoTokenizer
 
+from src.display.utils import parse_iso8601_datetime
 from src.envs import HAS_HIGHER_RATE_LIMIT
 
 
@@ -103,25 +104,29 @@ def get_model_arch(model_info: ModelInfo):
 
 
 def user_submission_permission(org_or_user, users_to_submission_dates, rate_limit_period, rate_limit_quota):
+    # Increase quota first if user has higher limits
+    if org_or_user in HAS_HIGHER_RATE_LIMIT:
+        rate_limit_quota *= 2
+
     if org_or_user not in users_to_submission_dates:
         return True, ""
+
     submission_dates = sorted(users_to_submission_dates[org_or_user])
+    time_limit = datetime.now(timezone.utc) - timedelta(days=rate_limit_period)
 
-    time_limit = (datetime.now(timezone.utc) - timedelta(days=rate_limit_period)).strftime("%Y-%m-%dT%H:%M:%SZ")
-    submissions_after_timelimit = [d for d in submission_dates if d > time_limit]
+    submissions_after_timelimit = [
+        parse_iso8601_datetime(d) for d in submission_dates
+        if parse_iso8601_datetime(d) > time_limit
+    ]
 
     num_models_submitted_in_period = len(submissions_after_timelimit)
-    if org_or_user in HAS_HIGHER_RATE_LIMIT:
-        rate_limit_quota = 2 * rate_limit_quota
-
-    if num_models_submitted_in_period > rate_limit_quota:
-        error_msg = f"Organisation or user `{org_or_user}`"
-        error_msg += f"already has {num_models_submitted_in_period} model requests submitted to the leaderboard "
-        error_msg += f"in the last {rate_limit_period} days.\n"
-        error_msg += (
-            "Please wait a couple of days before resubmitting, so that everybody can enjoy using the leaderboard 🤗"
-        )
+
+    # Use >= to correctly enforce the rate limit
+    if num_models_submitted_in_period >= rate_limit_quota:
+        error_msg = f"Organisation or user `{org_or_user}` already has {num_models_submitted_in_period} model requests submitted in the last {rate_limit_period} days.\n"
+        error_msg += "Please wait a couple of days before resubmitting, so that everybody can enjoy using the leaderboard 🤗"
         return False, error_msg
+
     return True, ""
 
 

src/submission/submit.py

@@ -63,11 +63,6 @@ def add_new_eval(
     if model in DO_NOT_SUBMIT_MODELS or base_model in DO_NOT_SUBMIT_MODELS:
         return styled_warning("Model authors have requested that their model be not submitted on the leaderboard.")
 
-    if model == "CohereForAI/c4ai-command-r-plus":
-        return styled_warning(
-            "This model cannot be submitted manually on the leaderboard before the transformers release."
-        )
-
     # Does the model actually exist?
     if revision == "":
         revision = "main"

tests/submission/test_user_submission_permission.py

@@ -0,0 +1,98 @@
+import unittest
+from unittest.mock import patch
+from datetime import datetime, timedelta, timezone
+
+from src.submission.check_validity import user_submission_permission
+from src.envs import RATE_LIMIT_PERIOD, RATE_LIMIT_QUOTA
+
+class TestUserSubmissionPermission(unittest.TestCase):
+
+    def setUp(self):
+        self.user_name = "test_user"
+        self.rate_limit_period = RATE_LIMIT_PERIOD
+        self.rate_limit_quota = RATE_LIMIT_QUOTA
+        self.fixed_now = datetime(2023, 6, 1, 12, 0, 0, tzinfo=timezone.utc)
+        # Submission dates that simulate various test cases
+        self.users_to_submission_dates = {
+            "test_user": [
+                (self.fixed_now - timedelta(days=1)).isoformat(),
+                (self.fixed_now - timedelta(days=2)).isoformat(),
+                (self.fixed_now - timedelta(days=3)).isoformat(),
+                (self.fixed_now - timedelta(days=4)).isoformat(),
+            ]
+        }
+
+    @staticmethod
+    def fixed_datetime_now(tz=None):
+        return datetime(2023, 6, 1, 12, 0, 0, tzinfo=timezone.utc)
+
+    @patch('src.submission.check_validity.datetime')
+    def test_user_below_quota(self, mock_datetime):
+        mock_datetime.now.side_effect = self.fixed_datetime_now
+        mock_datetime.fromisoformat = datetime.fromisoformat
+        allowed, message = user_submission_permission(
+            self.user_name, self.users_to_submission_dates, self.rate_limit_period, self.rate_limit_quota
+        )
+        self.assertTrue(allowed)
+
+    @patch('src.submission.check_validity.datetime')
+    def test_user_at_quota(self, mock_datetime):
+        mock_datetime.now.side_effect = self.fixed_datetime_now
+        mock_datetime.fromisoformat = datetime.fromisoformat
+        
+        # Add one more submission to reach the quota
+        self.users_to_submission_dates["test_user"].append(self.fixed_now.isoformat())
+        
+        allowed, message = user_submission_permission(
+            self.user_name, self.users_to_submission_dates, self.rate_limit_period, self.rate_limit_quota
+        )
+        self.assertFalse(allowed)
+        expected_message = (
+            f"Organisation or user `{self.user_name}` already has {self.rate_limit_quota} model requests submitted "
+            f"in the last {self.rate_limit_period} days.\n"
+            "Please wait a couple of days before resubmitting, so that everybody can enjoy using the leaderboard 🤗"
+        )
+        self.assertEqual(message, expected_message)
+
+    @patch('src.submission.check_validity.datetime')
+    def test_user_above_quota(self, mock_datetime):
+        mock_datetime.now.side_effect = self.fixed_datetime_now
+        mock_datetime.fromisoformat = datetime.fromisoformat
+        # Add more than quota submissions
+        for _ in range(self.rate_limit_quota + 1):
+            self.users_to_submission_dates["test_user"].append(self.fixed_now.isoformat())
+        allowed, message = user_submission_permission(
+            self.user_name, self.users_to_submission_dates, self.rate_limit_period, self.rate_limit_quota
+        )
+        self.assertFalse(allowed)
+
+    def test_user_no_previous_submissions(self):
+        allowed, message = user_submission_permission(
+            "new_user", self.users_to_submission_dates, self.rate_limit_period, self.rate_limit_quota
+        )
+        self.assertTrue(allowed)
+
+    @patch('src.submission.check_validity.HAS_HIGHER_RATE_LIMIT', ["specific_user"])
+    @patch('src.submission.check_validity.datetime')
+    def test_user_higher_rate_limit(self, mock_datetime):
+        mock_datetime.now.side_effect = self.fixed_datetime_now
+        mock_datetime.fromisoformat = datetime.fromisoformat
+        self.users_to_submission_dates["specific_user"] = [self.fixed_now.isoformat()] * (self.rate_limit_quota + 1)
+        allowed, message = user_submission_permission(
+            "specific_user", self.users_to_submission_dates, self.rate_limit_period, self.rate_limit_quota
+        )
+        self.assertTrue(allowed)
+
+    @patch('src.submission.check_validity.datetime')
+    def test_submission_just_outside_window(self, mock_datetime):
+        mock_datetime.now.side_effect = self.fixed_datetime_now
+        mock_datetime.fromisoformat = datetime.fromisoformat
+        old_submission = (self.fixed_now - timedelta(days=self.rate_limit_period, seconds=1)).isoformat()
+        self.users_to_submission_dates["test_user"] = [old_submission]
+        allowed, message = user_submission_permission(
+            self.user_name, self.users_to_submission_dates, self.rate_limit_period, self.rate_limit_quota
+        )
+        self.assertTrue(allowed)
+
+if __name__ == '__main__':
+    unittest.main()