Simplified leaderboard v0

app.py

@@ -25,16 +25,9 @@ from src.display.utils import (
     WeightType,
     Precision
 )
-from src.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, H4_TOKEN, IS_PUBLIC, QUEUE_REPO, REPO_ID, RESULTS_REPO
+from src.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, H4_TOKEN, QUEUE_REPO, REPO_ID, RESULTS_REPO
 from src.populate import get_evaluation_queue_df, get_leaderboard_df
 from src.submission.submit import add_new_eval
-from src.submission.check_validity import already_submitted_models
-from src.tools.collections import update_collections
-from src.tools.plots import (
-    create_metric_plot_obj,
-    create_plot_df,
-    create_scores_df,
-)
 
 
 def restart_space():
@@ -57,11 +50,8 @@ except Exception:
 
 
 raw_data, original_df = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, COLS, BENCHMARK_COLS)
-update_collections(original_df.copy())
 leaderboard_df = original_df.copy()
 
-plot_df = create_plot_df(create_scores_df(raw_data))
-
 (
     finished_eval_queue_df,
     running_eval_queue_df,
@@ -251,22 +241,6 @@ with demo:
                     queue=True,
                 )
 
-        with gr.TabItem("📈 Metrics through time", elem_id="llm-benchmark-tab-table", id=4):
-            with gr.Row():
-                with gr.Column():
-                    chart = create_metric_plot_obj(
-                        plot_df,
-                        [AutoEvalColumn.average.name],
-                        title="Average of Top Scores and Human Baseline Over Time (from last update)",
-                    )
-                    gr.Plot(value=chart, min_width=500) 
-                with gr.Column():
-                    chart = create_metric_plot_obj(
-                        plot_df,
-                        BENCHMARK_COLS,
-                        title="Top Scores and Human Baseline Over Time (from last update)",
-                    )
-                    gr.Plot(value=chart, min_width=500) 
         with gr.TabItem("📝 About", elem_id="llm-benchmark-tab-table", id=2):
             gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")
 
@@ -317,7 +291,6 @@ with demo:
                 with gr.Column():
                     model_name_textbox = gr.Textbox(label="Model name")
                     revision_name_textbox = gr.Textbox(label="Revision commit", placeholder="main")
-                    private = gr.Checkbox(False, label="Private", visible=not IS_PUBLIC)
                     model_type = gr.Dropdown(
                         choices=[t.to_str(" : ") for t in ModelType if t != ModelType.Unknown],
                         label="Model type",
@@ -352,7 +325,6 @@ with demo:
                     base_model_name_textbox,
                     revision_name_textbox,
                     precision,
-                    private,
                     weight_type,
                     model_type,
                 ],

requirements.txt

@@ -5,15 +5,11 @@ datasets==2.14.5
 gradio==4.4.0
 gradio_client==0.7.0
 huggingface-hub>=0.18.0
-markdown-it-py==2.2.0
-MarkupSafe==2.1.2
 matplotlib==3.7.1
 numpy==1.24.2
 pandas==2.0.0
-plotly==5.14.1
 python-dateutil==2.8.2
 requests==2.28.2
-semantic-version==2.10.0
 tqdm==4.65.0
 transformers==4.35.2
 tokenizers>=0.15.0

src/display/about.py

@@ -1,85 +1,34 @@
 from src.display.utils import ModelType
 
-TITLE = """<h1 align="center" id="space-title">🤗 Open LLM Leaderboard</h1>"""
+# To complete, what is your leaderboard name
+TITLE = """<h1 align="center" id="space-title">Leaderboard</h1>"""
 
+# to complete - what does your leaderboard evaluate
 INTRODUCTION_TEXT = """
-📐 The 🤗 Open LLM Leaderboard aims to track, rank and evaluate open LLMs and chatbots.
-
-🤗 Submit a model for automated evaluation on the 🤗 GPU cluster on the "Submit" page!
-The leaderboard's backend runs the great [Eleuther AI Language Model Evaluation Harness](https://github.com/EleutherAI/lm-evaluation-harness) - read more details in the "About" page!
 """
 
+# to complete - which evaluations are you running? how can people reproduce what you have?
 LLM_BENCHMARKS_TEXT = f"""
-Useful links: [FAQ](https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/discussions/179), [Community resources](https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/discussions/174), [Collection of best models](https://huggingface.co/collections/open-llm-leaderboard/llm-leaderboard-best-models-652d6c7965a4619fb5c27a03).
-
-# Context
-With the plethora of large language models (LLMs) and chatbots being released week upon week, often with grandiose claims of their performance, it can be hard to filter out the genuine progress that is being made by the open-source community and which model is the current state of the art.
-
-## Icons
-- {ModelType.PT.to_str(" : ")} model: new, base models, trained on a given corpora
-- {ModelType.FT.to_str(" : ")} model: pretrained models finetuned on more data
-Specific fine-tune subcategories (more adapted to chat):
-- {ModelType.IFT.to_str(" : ")} model: instruction fine-tunes, which are model fine-tuned specifically on datasets of task instruction 
-- {ModelType.RL.to_str(" : ")} model: reinforcement fine-tunes, which usually change the model loss a bit with an added policy. 
-If there is no icon, we have not uploaded the information on the model yet, feel free to open an issue with the model information!
-
-"Flagged" indicates that this model has been flagged by the community, and should probably be ignored! Clicking the link will redirect you to the discussion about the model.
-(For ex, the model was trained on the evaluation data, and is therefore cheating on the leaderboard.)
-
 ## How it works
 
-📈 We evaluate models on 7 key benchmarks using the <a href="https://github.com/EleutherAI/lm-evaluation-harness" target="_blank">  Eleuther AI Language Model Evaluation Harness </a>, a unified framework to test generative language models on a large number of different evaluation tasks.
-
-- <a href="https://arxiv.org/abs/1803.05457" target="_blank">  AI2 Reasoning Challenge </a> (25-shot) - a set of grade-school science questions.
-- <a href="https://arxiv.org/abs/1905.07830" target="_blank">  HellaSwag </a> (10-shot) - a test of commonsense inference, which is easy for humans (~95%) but challenging for SOTA models.
-- <a href="https://arxiv.org/abs/2009.03300" target="_blank">  MMLU </a>  (5-shot) - a test to measure a text model's multitask accuracy. The test covers 57 tasks including elementary mathematics, US history, computer science, law, and more.
-- <a href="https://arxiv.org/abs/2109.07958" target="_blank">  TruthfulQA </a> (0-shot) - a test to measure a model's propensity to reproduce falsehoods commonly found online. Note: TruthfulQA in the Harness is actually a minima a 6-shots task, as it is prepended by 6 examples systematically, even when launched using 0 for the number of few-shot examples.
-- <a href="https://arxiv.org/abs/1907.10641" target="_blank">  Winogrande </a> (5-shot) - an adversarial and difficult Winograd benchmark at scale, for commonsense reasoning.
-- <a href="https://arxiv.org/abs/2110.14168" target="_blank">  GSM8k </a> (5-shot) - diverse grade school math word problems to measure a model's ability to solve multi-step mathematical reasoning problems.
-- <a href="https://arxiv.org/abs/1903.00161" target="_blank">  DROP </a> (3-shot) - English reading comprehension benchmark requiring Discrete Reasoning Over the content of Paragraphs.
-
-For all these evaluations, a higher score is a better score.
-We chose these benchmarks as they test a variety of reasoning and general knowledge across a wide variety of fields in 0-shot and few-shot settings.
-
-## Details and logs
-You can find:
-- detailed numerical results in the `results` Hugging Face dataset: https://huggingface.co/datasets/open-llm-leaderboard/results
-- details on the input/outputs for the models in the `details` of each model, that you can access by clicking the 📄 emoji after the model name
-- community queries and running status in the `requests` Hugging Face dataset: https://huggingface.co/datasets/open-llm-leaderboard/requests
-
 ## Reproducibility
-To reproduce our results, here is the commands you can run, using [this version](https://github.com/EleutherAI/lm-evaluation-harness/tree/b281b0921b636bc36ad05c0b0b0763bd6dd43463) of the Eleuther AI Harness:
-`python main.py --model=hf-causal --model_args="pretrained=<your_model>,use_accelerate=True,revision=<your_model_revision>"`
-` --tasks=<task_list> --num_fewshot=<n_few_shot> --batch_size=2 --output_path=<output_path>`
-
-The total batch size we get for models which fit on one A100 node is 16 (8 GPUs * 2). If you don't use parallelism, adapt your batch size to fit.
-*You can expect results to vary slightly for different batch sizes because of padding.*
-
-The tasks and few shots parameters are:
-- ARC: 25-shot, *arc-challenge* (`acc_norm`)
-- HellaSwag: 10-shot, *hellaswag* (`acc_norm`)
-- TruthfulQA: 0-shot, *truthfulqa-mc* (`mc2`)
-- MMLU: 5-shot, *hendrycksTest-abstract_algebra,hendrycksTest-anatomy,hendrycksTest-astronomy,hendrycksTest-business_ethics,hendrycksTest-clinical_knowledge,hendrycksTest-college_biology,hendrycksTest-college_chemistry,hendrycksTest-college_computer_science,hendrycksTest-college_mathematics,hendrycksTest-college_medicine,hendrycksTest-college_physics,hendrycksTest-computer_security,hendrycksTest-conceptual_physics,hendrycksTest-econometrics,hendrycksTest-electrical_engineering,hendrycksTest-elementary_mathematics,hendrycksTest-formal_logic,hendrycksTest-global_facts,hendrycksTest-high_school_biology,hendrycksTest-high_school_chemistry,hendrycksTest-high_school_computer_science,hendrycksTest-high_school_european_history,hendrycksTest-high_school_geography,hendrycksTest-high_school_government_and_politics,hendrycksTest-high_school_macroeconomics,hendrycksTest-high_school_mathematics,hendrycksTest-high_school_microeconomics,hendrycksTest-high_school_physics,hendrycksTest-high_school_psychology,hendrycksTest-high_school_statistics,hendrycksTest-high_school_us_history,hendrycksTest-high_school_world_history,hendrycksTest-human_aging,hendrycksTest-human_sexuality,hendrycksTest-international_law,hendrycksTest-jurisprudence,hendrycksTest-logical_fallacies,hendrycksTest-machine_learning,hendrycksTest-management,hendrycksTest-marketing,hendrycksTest-medical_genetics,hendrycksTest-miscellaneous,hendrycksTest-moral_disputes,hendrycksTest-moral_scenarios,hendrycksTest-nutrition,hendrycksTest-philosophy,hendrycksTest-prehistory,hendrycksTest-professional_accounting,hendrycksTest-professional_law,hendrycksTest-professional_medicine,hendrycksTest-professional_psychology,hendrycksTest-public_relations,hendrycksTest-security_studies,hendrycksTest-sociology,hendrycksTest-us_foreign_policy,hendrycksTest-virology,hendrycksTest-world_religions* (average of all the results `acc`)
-- Winogrande: 5-shot, *winogrande* (`acc`)
-- GSM8k: 5-shot, *gsm8k* (`acc`)
-- DROP: 3-shot, *drop* (`f1`)
-
-Side note on the baseline scores: 
-- for log-likelihood evaluation, we select the random baseline
-- for DROP, we select the best submission score according to [their leaderboard](https://leaderboard.allenai.org/drop/submissions/public) when the paper came out (NAQANet score)
-- for GSM8K, we select the score obtained in the paper after finetuning a 6B model on the full GSM8K training set for 50 epochs
+To reproduce our results, here is the commands you can run:
 
 ## Quantization
 To get more information about quantization, see:
 - 8 bits: [blog post](https://huggingface.co/blog/hf-bitsandbytes-integration), [paper](https://arxiv.org/abs/2208.07339)
 - 4 bits: [blog post](https://huggingface.co/blog/4bit-transformers-bitsandbytes), [paper](https://arxiv.org/abs/2305.14314)
+
+## Model types
+- {ModelType.PT.to_str(" : ")} model: new, base models, trained on a given corpora
+- {ModelType.FT.to_str(" : ")} model: pretrained models finetuned on more data
+Specific fine-tune subcategories (more adapted to chat):
+- {ModelType.IFT.to_str(" : ")} model: instruction fine-tunes, which are model fine-tuned specifically on datasets of task instruction 
+- {ModelType.RL.to_str(" : ")} model: reinforcement fine-tunes, which usually change the model loss a bit with an added policy. 
+If there is no icon, we have not uploaded the information on the model yet, feel free to open an issue with the model information!
 """
 
 EVALUATION_QUEUE_TEXT = """
-# Evaluation Queue for the 🤗 Open LLM Leaderboard
-
-Models added here will be automatically evaluated on the 🤗 cluster.
-
 ## Some good practices before submitting a model
 
 ### 1) Make sure you can load your model and tokenizer using AutoClasses:
@@ -111,109 +60,4 @@ If everything is done, check you can launch the EleutherAIHarness on your model
 
 CITATION_BUTTON_LABEL = "Copy the following snippet to cite these results"
 CITATION_BUTTON_TEXT = r"""
-@misc{open-llm-leaderboard,
-  author = {Edward Beeching and Clémentine Fourrier and Nathan Habib and Sheon Han and Nathan Lambert and Nazneen Rajani and Omar Sanseviero and Lewis Tunstall and Thomas Wolf},
-  title = {Open LLM Leaderboard},
-  year = {2023},
-  publisher = {Hugging Face},
-  howpublished = "\url{https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard}"
-}
-@software{eval-harness,
-  author       = {Gao, Leo and
-                  Tow, Jonathan and
-                  Biderman, Stella and
-                  Black, Sid and
-                  DiPofi, Anthony and
-                  Foster, Charles and
-                  Golding, Laurence and
-                  Hsu, Jeffrey and
-                  McDonell, Kyle and
-                  Muennighoff, Niklas and
-                  Phang, Jason and
-                  Reynolds, Laria and
-                  Tang, Eric and
-                  Thite, Anish and
-                  Wang, Ben and
-                  Wang, Kevin and
-                  Zou, Andy},
-  title        = {A framework for few-shot language model evaluation},
-  month        = sep,
-  year         = 2021,
-  publisher    = {Zenodo},
-  version      = {v0.0.1},
-  doi          = {10.5281/zenodo.5371628},
-  url          = {https://doi.org/10.5281/zenodo.5371628}
-}
-@misc{clark2018think,
-      title={Think you have Solved Question Answering? Try ARC, the AI2 Reasoning Challenge},
-      author={Peter Clark and Isaac Cowhey and Oren Etzioni and Tushar Khot and Ashish Sabharwal and Carissa Schoenick and Oyvind Tafjord},
-      year={2018},
-      eprint={1803.05457},
-      archivePrefix={arXiv},
-      primaryClass={cs.AI}
-}
-@misc{zellers2019hellaswag,
-      title={HellaSwag: Can a Machine Really Finish Your Sentence?},
-      author={Rowan Zellers and Ari Holtzman and Yonatan Bisk and Ali Farhadi and Yejin Choi},
-      year={2019},
-      eprint={1905.07830},
-      archivePrefix={arXiv},
-      primaryClass={cs.CL}
-}
-@misc{hendrycks2021measuring,
-      title={Measuring Massive Multitask Language Understanding},
-      author={Dan Hendrycks and Collin Burns and Steven Basart and Andy Zou and Mantas Mazeika and Dawn Song and Jacob Steinhardt},
-      year={2021},
-      eprint={2009.03300},
-      archivePrefix={arXiv},
-      primaryClass={cs.CY}
-}
-@misc{lin2022truthfulqa,
-      title={TruthfulQA: Measuring How Models Mimic Human Falsehoods},
-      author={Stephanie Lin and Jacob Hilton and Owain Evans},
-      year={2022},
-      eprint={2109.07958},
-      archivePrefix={arXiv},
-      primaryClass={cs.CL}
-}
-@misc{DBLP:journals/corr/abs-1907-10641,
-      title={{WINOGRANDE:} An Adversarial Winograd Schema Challenge at Scale},
-      author={Keisuke Sakaguchi and Ronan Le Bras and Chandra Bhagavatula and Yejin Choi},
-      year={2019},
-      eprint={1907.10641},
-      archivePrefix={arXiv},
-      primaryClass={cs.CL}
-}
-@misc{DBLP:journals/corr/abs-2110-14168,
-      title={Training Verifiers to Solve Math Word Problems},
-      author={Karl Cobbe and
-                  Vineet Kosaraju and
-                  Mohammad Bavarian and
-                  Mark Chen and
-                  Heewoo Jun and
-                  Lukasz Kaiser and
-                  Matthias Plappert and
-                  Jerry Tworek and
-                  Jacob Hilton and
-                  Reiichiro Nakano and
-                  Christopher Hesse and
-                  John Schulman},
-      year={2021},
-      eprint={2110.14168},
-      archivePrefix={arXiv},
-      primaryClass={cs.CL}
-}
-@misc{DBLP:journals/corr/abs-1903-00161,
-      title={{DROP:} {A} Reading Comprehension Benchmark Requiring Discrete Reasoning
-                  Over Paragraphs},
-      author={Dheeru Dua and
-                  Yizhong Wang and
-                  Pradeep Dasigi and
-                  Gabriel Stanovsky and
-                  Sameer Singh and
-                  Matt Gardner},
-      year={2019},
-      eprinttype={arXiv},
-      eprint={1903.00161},
-      primaryClass={cs.CL}
-}"""
+"""

src/display/formatting.py

@@ -13,11 +13,7 @@ def model_hyperlink(link, model_name):
 
 def make_clickable_model(model_name):
     link = f"https://huggingface.co/{model_name}"
-
-    details_model_name = model_name.replace("/", "__")
-    details_link = f"https://huggingface.co/datasets/open-llm-leaderboard/details_{details_model_name}"
-
-    return model_hyperlink(link, model_name) + "  " + model_hyperlink(details_link, "📑")
+    return model_hyperlink(link, model_name)
 
 
 def styled_error(error):

src/display/utils.py

@@ -13,14 +13,10 @@ class Task:
     metric: str
     col_name: str
 
+# Init: to update with your specific keys
 class Tasks(Enum):
-    arc = Task("arc:challenge", "acc_norm", "ARC")
-    hellaswag = Task("hellaswag", "acc_norm", "HellaSwag")
-    mmlu = Task("hendrycksTest", "acc", "MMLU")
-    truthfulqa = Task("truthfulqa:mc", "mc2", "TruthfulQA")
-    winogrande = Task("winogrande", "acc", "Winogrande")
-    gsm8k = Task("gsm8k", "acc", "GSM8K")
-    drop = Task("drop", "f1", "DROP")
+    task0 = Task("Key in the harness", "metric in the harness", "display name")
+    task1 = Task("Key in the harness", "metric in the harness", "display name")
 
 # These classes are for user facing column names,
 # to avoid having to change them all around the code
@@ -67,44 +63,20 @@ class EvalQueueColumn:  # Queue column
     weight_type = ColumnContent("weight_type", "str", "Original")
     status = ColumnContent("status", "str", True)
 
-
 baseline_row = {
     AutoEvalColumn.model.name: "<p>Baseline</p>",
     AutoEvalColumn.revision.name: "N/A",
     AutoEvalColumn.precision.name: None,
-    AutoEvalColumn.average.name: 31.0,
-    AutoEvalColumn.arc.name: 25.0,
-    AutoEvalColumn.hellaswag.name: 25.0,
-    AutoEvalColumn.mmlu.name: 25.0,
-    AutoEvalColumn.truthfulqa.name: 25.0,
-    AutoEvalColumn.winogrande.name: 50.0,
-    AutoEvalColumn.gsm8k.name: 0.21,
-    AutoEvalColumn.drop.name: 0.47,
+    AutoEvalColumn.average.name: 0,
     AutoEvalColumn.dummy.name: "baseline",
     AutoEvalColumn.model_type.name: "",
 }
 
-# Average ⬆️ human baseline is 0.897 (source: averaging human baselines below)
-# ARC human baseline is 0.80 (source: https://lab42.global/arc/)
-# HellaSwag human baseline is 0.95 (source: https://deepgram.com/learn/hellaswag-llm-benchmark-guide)
-# MMLU human baseline is 0.898 (source: https://openreview.net/forum?id=d7KBjmI3GmQ)
-# TruthfulQA human baseline is 0.94(source: https://arxiv.org/pdf/2109.07958.pdf)
-# Drop: https://leaderboard.allenai.org/drop/submissions/public
-# Winogrande: https://leaderboard.allenai.org/winogrande/submissions/public
-# GSM8K: paper
-# Define the human baselines
 human_baseline_row = {
     AutoEvalColumn.model.name: "<p>Human performance</p>",
     AutoEvalColumn.revision.name: "N/A",
     AutoEvalColumn.precision.name: None,
-    AutoEvalColumn.average.name: 92.75,
-    AutoEvalColumn.arc.name: 80.0,
-    AutoEvalColumn.hellaswag.name: 95.0,
-    AutoEvalColumn.mmlu.name: 89.8,
-    AutoEvalColumn.truthfulqa.name: 94.0,
-    AutoEvalColumn.winogrande.name: 94.0,
-    AutoEvalColumn.gsm8k.name: 100,
-    AutoEvalColumn.drop.name: 96.42,
+    AutoEvalColumn.average.name: 0,
     AutoEvalColumn.dummy.name: "human_baseline",
     AutoEvalColumn.model_type.name: "",
 }
@@ -112,7 +84,8 @@ human_baseline_row = {
 @dataclass
 class ModelDetails:
     name: str
-    symbol: str = "" # emoji, only for the model type
+    display_name: str = ""
+    symbol: str = "" # emoji
 
 
 class ModelType(Enum):
@@ -162,9 +135,6 @@ class Precision(Enum):
         if precision in ["GPTQ", "None"]:
             return Precision.qt_GPTQ
         return Precision.Unknown
-        
-
-
 
 # Column selection
 COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden]

src/envs.py

@@ -5,28 +5,15 @@ from huggingface_hub import HfApi
 # clone / pull the lmeh eval data
 H4_TOKEN = os.environ.get("H4_TOKEN", None)
 
-REPO_ID = "HuggingFaceH4/open_llm_leaderboard"
-QUEUE_REPO = "open-llm-leaderboard/requests"
-RESULTS_REPO = "open-llm-leaderboard/results"
-
-PRIVATE_QUEUE_REPO = "open-llm-leaderboard/private-requests"
-PRIVATE_RESULTS_REPO = "open-llm-leaderboard/private-results"
-
-IS_PUBLIC = bool(os.environ.get("IS_PUBLIC", True))
+OWNER = "clefourrier"
+REPO_ID = f"{OWNER}/leaderboard"
+QUEUE_REPO = f"{OWNER}/requests"
+RESULTS_REPO = f"{OWNER}/results"
 
 CACHE_PATH=os.getenv("HF_HOME", ".")
 
+# Local caches
 EVAL_REQUESTS_PATH = os.path.join(CACHE_PATH, "eval-queue")
 EVAL_RESULTS_PATH = os.path.join(CACHE_PATH, "eval-results")
 
-EVAL_REQUESTS_PATH_PRIVATE = "eval-queue-private"
-EVAL_RESULTS_PATH_PRIVATE = "eval-results-private"
-
-PATH_TO_COLLECTION = "open-llm-leaderboard/llm-leaderboard-best-models-652d6c7965a4619fb5c27a03"
-
-# Rate limit variables
-RATE_LIMIT_PERIOD = 7
-RATE_LIMIT_QUOTA = 5
-HAS_HIGHER_RATE_LIMIT = ["TheBloke"]
-
 API = HfApi(token=H4_TOKEN)

src/leaderboard/read_evals.py

@@ -72,23 +72,8 @@ class EvalResult:
         results = {}
         for task in Tasks:
             task = task.value
-            # We skip old mmlu entries
-            wrong_mmlu_version = False
-            if task.benchmark == "hendrycksTest":
-                for mmlu_k in ["harness|hendrycksTest-abstract_algebra|5", "hendrycksTest-abstract_algebra"]:
-                    if mmlu_k in data["versions"] and data["versions"][mmlu_k] == 0:
-                        wrong_mmlu_version = True
-
-            if wrong_mmlu_version:
-                continue
-
-            # Some truthfulQA values are NaNs
-            if task.benchmark == "truthfulqa:mc" and "harness|truthfulqa:mc|0" in data["results"]:
-                if math.isnan(float(data["results"]["harness|truthfulqa:mc|0"][task.metric])):
-                    results[task.benchmark] = 0.0
-                    continue
 
-            # We average all scores of a given metric (mostly for mmlu)
+            # We average all scores of a given metric
             accs = np.array([v.get(task.metric, None) for k, v in data["results"].items() if task.benchmark in k])
             if accs.size == 0 or any([acc is None for acc in accs]):
                 continue

src/submission/check_validity.py

@@ -10,13 +10,8 @@ from huggingface_hub.hf_api import ModelInfo
 from transformers import AutoConfig
 from transformers.models.auto.tokenization_auto import tokenizer_class_from_name, get_tokenizer_config
 
-from src.envs import HAS_HIGHER_RATE_LIMIT
-
-
-# ht to @Wauplin, thank you for the snippet!
-# See https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/discussions/317
 def check_model_card(repo_id: str) -> tuple[bool, str]:
-    # Returns operation status, and error message
+    """Checks if the model card and license exist and have been filled"""
     try:
         card = ModelCard.load(repo_id)
     except huggingface_hub.utils.EntryNotFoundError:
@@ -38,6 +33,7 @@ def check_model_card(repo_id: str) -> tuple[bool, str]:
 
 
 def is_model_on_hub(model_name: str, revision: str, token: str = None, trust_remote_code=False, test_tokenizer=False) -> tuple[bool, str]:
+    """Makes sure the model is on the hub, and uses a valid configuration (in the latest transformers version)"""
     try:
         config = AutoConfig.from_pretrained(model_name, revision=revision, trust_remote_code=trust_remote_code, token=token)
         if test_tokenizer:
@@ -69,47 +65,20 @@ def is_model_on_hub(model_name: str, revision: str, token: str = None, trust_rem
 
 
 def get_model_size(model_info: ModelInfo, precision: str):
-    size_pattern = size_pattern = re.compile(r"(\d\.)?\d+(b|m)")
+    """Gets the model size from the configuration, or the model name if the configuration does not contain the information."""
     try:
         model_size = round(model_info.safetensors["total"] / 1e9, 3)
-    except (AttributeError, TypeError ):
-        try:
-            size_match = re.search(size_pattern, model_info.modelId.lower())
-            model_size = size_match.group(0)
-            model_size = round(float(model_size[:-1]) if model_size[-1] == "b" else float(model_size[:-1]) / 1e3, 3)
-        except AttributeError:
-            return 0  # Unknown model sizes are indicated as 0, see NUMERIC_INTERVALS in app.py
+    except (AttributeError, TypeError):
+        return 0  # Unknown model sizes are indicated as 0, see NUMERIC_INTERVALS in app.py
 
     size_factor = 8 if (precision == "GPTQ" or "gptq" in model_info.modelId.lower()) else 1
     model_size = size_factor * model_size
     return model_size
 
 def get_model_arch(model_info: ModelInfo):
+    """Gets the model architecture from the configuration"""
     return model_info.config.get("architectures", "Unknown")
 
-def user_submission_permission(org_or_user, users_to_submission_dates, rate_limit_period, rate_limit_quota):
-    if org_or_user not in users_to_submission_dates:
-        return True, ""
-    submission_dates = sorted(users_to_submission_dates[org_or_user])
-
-    time_limit = (datetime.now(timezone.utc) - timedelta(days=rate_limit_period)).strftime("%Y-%m-%dT%H:%M:%SZ")
-    submissions_after_timelimit = [d for d in submission_dates if d > time_limit]
-
-    num_models_submitted_in_period = len(submissions_after_timelimit)
-    if org_or_user in HAS_HIGHER_RATE_LIMIT:
-        rate_limit_quota = 2 * rate_limit_quota
-
-    if num_models_submitted_in_period > rate_limit_quota:
-        error_msg = f"Organisation or user `{org_or_user}`"
-        error_msg += f"already has {num_models_submitted_in_period} model requests submitted to the leaderboard "
-        error_msg += f"in the last {rate_limit_period} days.\n"
-        error_msg += (
-            "Please wait a couple of days before resubmitting, so that everybody can enjoy using the leaderboard 🤗"
-        )
-        return False, error_msg
-    return True, ""
-
-
 def already_submitted_models(requested_models_dir: str) -> set[str]:
     depth = 1
     file_names = []

src/submission/submit.py

@@ -3,14 +3,12 @@ import os
 from datetime import datetime, timezone
 
 from src.display.formatting import styled_error, styled_message, styled_warning
-from src.envs import API, EVAL_REQUESTS_PATH, H4_TOKEN, QUEUE_REPO, RATE_LIMIT_PERIOD, RATE_LIMIT_QUOTA
-from src.leaderboard.filter_models import DO_NOT_SUBMIT_MODELS
+from src.envs import API, EVAL_REQUESTS_PATH, H4_TOKEN, QUEUE_REPO
 from src.submission.check_validity import (
     already_submitted_models,
     check_model_card,
     get_model_size,
     is_model_on_hub,
-    user_submission_permission,
 )
 
 REQUESTED_MODELS = None
@@ -21,7 +19,6 @@ def add_new_eval(
     base_model: str,
     revision: str,
     precision: str,
-    private: bool,
     weight_type: str,
     model_type: str,
 ):
@@ -42,18 +39,6 @@ def add_new_eval(
     if model_type is None or model_type == "":
         return styled_error("Please select a model type.")
 
-    # Is the user rate limited?
-    if user_name != "":
-        user_can_submit, error_msg = user_submission_permission(
-            user_name, USERS_TO_SUBMISSION_DATES, RATE_LIMIT_PERIOD, RATE_LIMIT_QUOTA
-        )
-        if not user_can_submit:
-            return styled_error(error_msg)
-
-    # Did the model authors forbid its submission to the leaderboard?
-    if model in DO_NOT_SUBMIT_MODELS or base_model in DO_NOT_SUBMIT_MODELS:
-        return styled_warning("Model authors have requested that their model be not submitted on the leaderboard.")
-
     # Does the model actually exist?
     if revision == "":
         revision = "main"
@@ -94,7 +79,6 @@ def add_new_eval(
         "model": model,
         "base_model": base_model,
         "revision": revision,
-        "private": private,
         "precision": precision,
         "weight_type": weight_type,
         "status": "PENDING",
@@ -112,7 +96,7 @@ def add_new_eval(
     print("Creating eval file")
     OUT_DIR = f"{EVAL_REQUESTS_PATH}/{user_name}"
     os.makedirs(OUT_DIR, exist_ok=True)
-    out_path = f"{OUT_DIR}/{model_path}_eval_request_{private}_{precision}_{weight_type}.json"
+    out_path = f"{OUT_DIR}/{model_path}_eval_request_False_{precision}_{weight_type}.json"
 
     with open(out_path, "w") as f:
         f.write(json.dumps(eval_entry))