refacto style + rate limit

app.py

@@ -6,18 +6,6 @@ import pandas as pd
 from apscheduler.schedulers.background import BackgroundScheduler
 from huggingface_hub import snapshot_download
 
-from src.display.utils import (
-    COLS,
-    TYPES,
-    BENCHMARK_COLS,
-    EVAL_COLS,
-    EVAL_TYPES,
-    AutoEvalColumn,
-    ModelType,
-    NUMERIC_INTERVALS,
-    fields,
-)
-from src.display.css_html_js import custom_css, get_window_url_params
 from src.display.about import (
     CITATION_BUTTON_LABEL,
     CITATION_BUTTON_TEXT,
@@ -26,17 +14,29 @@ from src.display.about import (
     LLM_BENCHMARKS_TEXT,
     TITLE,
 )
+from src.display.css_html_js import custom_css, get_window_url_params
+from src.display.utils import (
+    BENCHMARK_COLS,
+    COLS,
+    EVAL_COLS,
+    EVAL_TYPES,
+    NUMERIC_INTERVALS,
+    TYPES,
+    AutoEvalColumn,
+    ModelType,
+    fields,
+)
+from src.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, H4_TOKEN, IS_PUBLIC, QUEUE_REPO, REPO_ID, RESULTS_REPO
+from src.populate import get_evaluation_queue_df, get_leaderboard_df
+from src.submission.submit import add_new_eval
+from src.tools.collections import update_collections
 from src.tools.plots import (
+    HUMAN_BASELINES,
     create_metric_plot_obj,
-    create_scores_df,
     create_plot_df,
+    create_scores_df,
     join_model_info_with_results,
-    HUMAN_BASELINES,
 )
-from src.tools.collections import update_collections
-from src.populate import get_evaluation_queue_df, get_leaderboard_df
-from src.envs import H4_TOKEN, QUEUE_REPO, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, RESULTS_REPO, API, REPO_ID, IS_PUBLIC
-from src.submission.submit import add_new_eval
 
 
 def restart_space():
@@ -61,9 +61,9 @@ original_df = get_leaderboard_df(EVAL_RESULTS_PATH, COLS, BENCHMARK_COLS)
 update_collections(original_df.copy())
 leaderboard_df = original_df.copy()
 
-#models = original_df["model_name_for_query"].tolist()  # needed for model backlinks in their to the leaderboard
+# models = original_df["model_name_for_query"].tolist()  # needed for model backlinks in their to the leaderboard
 # plot_df = create_plot_df(create_scores_df(join_model_info_with_results(original_df)))
-#to_be_dumped = f"models = {repr(models)}\n"
+# to_be_dumped = f"models = {repr(models)}\n"
 
 (
     finished_eval_queue_df,
@@ -173,8 +173,16 @@ with demo:
                         )
                     with gr.Row():
                         shown_columns = gr.CheckboxGroup(
-                            choices=[c.name for c in fields(AutoEvalColumn) if not c.hidden and not c.never_hidden and not c.dummy],
-                            value=[c.name for c in fields(AutoEvalColumn) if c.displayed_by_default and not c.hidden and not c.never_hidden],
+                            choices=[
+                                c.name
+                                for c in fields(AutoEvalColumn)
+                                if not c.hidden and not c.never_hidden and not c.dummy
+                            ],
+                            value=[
+                                c.name
+                                for c in fields(AutoEvalColumn)
+                                if c.displayed_by_default and not c.hidden and not c.never_hidden
+                            ],
                             label="Select columns to show",
                             elem_id="column-select",
                             interactive=True,

scripts/create_request_file.py

@@ -1,11 +1,12 @@
-from datetime import datetime, timezone
 import json
 import os
+import pprint
 import re
+from datetime import datetime, timezone
+
 import click
-from huggingface_hub import HfApi, snapshot_download
 from colorama import Fore
-import pprint
+from huggingface_hub import HfApi, snapshot_download
 
 EVAL_REQUESTS_PATH = "eval-queue"
 QUEUE_REPO = "open-llm-leaderboard/requests"

src/display/formatting.py

@@ -1,4 +1,5 @@
 import os
+
 from huggingface_hub import HfApi
 
 API = HfApi()

src/display/utils.py

@@ -1,7 +1,8 @@
 from dataclasses import dataclass
-import pandas as pd
 from enum import Enum
 
+import pandas as pd
+
 
 # These classes are for user facing column names,
 # to avoid having to change them all around the code

src/envs.py

@@ -1,4 +1,5 @@
 import os
+
 from huggingface_hub import HfApi
 
 # clone / pull the lmeh eval data
@@ -24,5 +25,6 @@ PATH_TO_COLLECTION = "open-llm-leaderboard/llm-leaderboard-best-models-652d6c796
 # Rate limit variables
 RATE_LIMIT_PERIOD = 7
 RATE_LIMIT_QUOTA = 5
+HAS_HIGHER_RATE_LIMIT = ["TheBloke"]
 
 API = HfApi(token=H4_TOKEN)

src/leaderboard/read_evals.py

@@ -1,15 +1,15 @@
+import glob
 import json
-import os
 import math
-import glob
+import os
 from dataclasses import dataclass
 from typing import Dict, List, Tuple
 
 import dateutil
 import numpy as np
 
-from src.display.utils import AutoEvalColumn, ModelType, Tasks
 from src.display.formatting import make_clickable_model
+from src.display.utils import AutoEvalColumn, ModelType, Tasks
 from src.submission.check_validity import is_model_on_hub
 
 
@@ -56,7 +56,9 @@ class EvalResult:
             model = org_and_model[1]
             result_key = f"{org}_{model}_{precision}"
 
-        still_on_hub = is_model_on_hub("/".join(org_and_model), config.get("model_sha", "main"), trust_remote_code=True)[0]
+        still_on_hub = is_model_on_hub(
+            "/".join(org_and_model), config.get("model_sha", "main"), trust_remote_code=True
+        )[0]
 
         # Extract results available in this file (some results are split in several files)
         results = {}
@@ -73,8 +75,8 @@ class EvalResult:
                 continue
 
             # Some truthfulQA values are NaNs
-            if task.benchmark == "truthfulqa:mc" and 'harness|truthfulqa:mc|0' in data["results"]:
-                if math.isnan(float(data["results"]['harness|truthfulqa:mc|0'][task.metric])):
+            if task.benchmark == "truthfulqa:mc" and "harness|truthfulqa:mc|0" in data["results"]:
+                if math.isnan(float(data["results"]["harness|truthfulqa:mc|0"][task.metric])):
                     results[task.benchmark] = 0.0
                     continue
 
@@ -191,7 +193,7 @@ def get_eval_results(results_path: str) -> List[EvalResult]:
     for v in eval_results.values():
         try:
             results.append(v.to_dict())
-        except KeyError: # not all eval values present 
+        except KeyError:  # not all eval values present
             continue
 
     return results

src/populate.py

@@ -3,10 +3,10 @@ import os
 
 import pandas as pd
 
+from src.display.formatting import has_no_nan_values, make_clickable_model
+from src.display.utils import AutoEvalColumn, EvalQueueColumn, baseline_row
 from src.leaderboard.filter_models import filter_models
 from src.leaderboard.read_evals import get_eval_results
-from src.display.formatting import make_clickable_model, has_no_nan_values
-from src.display.utils import AutoEvalColumn, EvalQueueColumn, baseline_row
 
 
 def get_leaderboard_df(results_path: str, cols: list, benchmark_cols: list) -> pd.DataFrame:

src/submission/check_validity.py

@@ -1,13 +1,15 @@
-import huggingface_hub
-import os
 import json
+import os
 import re
 from collections import defaultdict
-from huggingface_hub.hf_api import ModelInfo
+from datetime import datetime, timedelta, timezone
+
+import huggingface_hub
 from huggingface_hub import ModelCard
+from huggingface_hub.hf_api import ModelInfo
 from transformers import AutoConfig
 
-from datetime import datetime, timedelta, timezone
+from src.envs import HAS_HIGHER_RATE_LIMIT
 
 
 # ht to @Wauplin, thank you for the snippet!
@@ -76,6 +78,9 @@ def user_submission_permission(submission_name, users_to_submission_dates, rate_
     submissions_after_timelimit = [d for d in submission_dates if d > time_limit]
 
     num_models_submitted_in_period = len(submissions_after_timelimit)
+    if org_or_user in HAS_HIGHER_RATE_LIMIT:
+        rate_limit_quota = 2 * rate_limit_quota
+
     if num_models_submitted_in_period > rate_limit_quota:
         error_msg = f"Organisation or user `{org_or_user}`"
         error_msg += f"already has {num_models_submitted_in_period} model requests submitted to the leaderboard "

src/submission/submit.py

@@ -1,17 +1,17 @@
-import os, json
-
+import json
+import os
 from datetime import datetime, timezone
 
-from src.display.formatting import styled_error, styled_warning, styled_message
+from src.display.formatting import styled_error, styled_message, styled_warning
+from src.envs import API, EVAL_REQUESTS_PATH, H4_TOKEN, QUEUE_REPO, RATE_LIMIT_PERIOD, RATE_LIMIT_QUOTA
 from src.leaderboard.filter_models import DO_NOT_SUBMIT_MODELS
 from src.submission.check_validity import (
-    user_submission_permission,
-    is_model_on_hub,
-    get_model_size,
-    check_model_card,
     already_submitted_models,
+    check_model_card,
+    get_model_size,
+    is_model_on_hub,
+    user_submission_permission,
 )
-from src.envs import RATE_LIMIT_QUOTA, RATE_LIMIT_PERIOD, H4_TOKEN, EVAL_REQUESTS_PATH, API, QUEUE_REPO
 
 requested_models, users_to_submission_dates = already_submitted_models(EVAL_REQUESTS_PATH)
 

src/tools/collections.py

@@ -1,11 +1,11 @@
 import os
+
 import pandas as pd
-from pandas import DataFrame
-from huggingface_hub import get_collection, add_collection_item, update_collection_item, delete_collection_item
+from huggingface_hub import add_collection_item, delete_collection_item, get_collection, update_collection_item
 from huggingface_hub.utils._errors import HfHubHTTPError
+from pandas import DataFrame
 
 from src.display.utils import AutoEvalColumn, ModelType
-
 from src.envs import H4_TOKEN, PATH_TO_COLLECTION
 
 # Specific intervals for the collections

src/tools/plots.py

@@ -1,9 +1,11 @@
+import pickle
+from datetime import datetime, timezone
+from typing import Any, Dict, List, Tuple
+
 import pandas as pd
 import plotly.express as px
 from plotly.graph_objs import Figure
-import pickle
-from datetime import datetime, timezone
-from typing import List, Dict, Tuple, Any
+
 from src.leaderboard.filter_models import FLAGGED_MODELS
 
 # Average ⬆️ human baseline is 0.897 (source: averaging human baselines below)