FT: precision and adapter models

app.py

@@ -28,7 +28,6 @@ PRIVATE_QUEUE_REPO = "open-llm-leaderboard/private-requests"
 PRIVATE_RESULTS_REPO = "open-llm-leaderboard/private-results"
 
 IS_PUBLIC = bool(os.environ.get("IS_PUBLIC", True))
-ADD_PLOTS = False
 
 EVAL_REQUESTS_PATH = "eval-queue"
 EVAL_RESULTS_PATH = "eval-results"
@@ -56,8 +55,8 @@ COLS_LITE = [c.name for c in fields(AutoEvalColumn) if c.displayed_by_default an
 TYPES_LITE = [c.type for c in fields(AutoEvalColumn) if c.displayed_by_default and not c.hidden]
 
 if not IS_PUBLIC:
-    COLS.insert(2, AutoEvalColumn.is_8bit.name)
-    TYPES.insert(2, AutoEvalColumn.is_8bit.type)
+    COLS.insert(2, AutoEvalColumn.precision.name)
+    TYPES.insert(2, AutoEvalColumn.precision.type)
 
 EVAL_COLS = [c.name for c in fields(EvalQueueColumn)]
 EVAL_TYPES = [c.type for c in fields(EvalQueueColumn)]
@@ -177,25 +176,27 @@ def add_new_eval(
     model: str,
     base_model: str,
     revision: str,
-    is_8_bit_eval: bool,
+    precision: str,
     private: bool,
-    is_delta_weight: bool,
+    weight_type: str,
 ):
+    precision = precision.split(" ")[0]
     current_time = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
 
     # check the model actually exists before adding the eval
     if revision == "":
         revision = "main"
 
-    if is_delta_weight: 
+    if weight_type in ["Delta", "Adapter"]:
         base_model_on_hub, error = is_model_on_hub(base_model, revision)
         if not base_model_on_hub:
             return styled_error(f'Base model "{base_model}" {error}')
+        
 
     model_on_hub, error = is_model_on_hub(model, revision)
     if not model_on_hub:
         return styled_error(f'Model "{model}" {error}')
-
+    
     print("adding new eval")
 
     eval_entry = {
@@ -203,8 +204,8 @@ def add_new_eval(
         "base_model": base_model,
         "revision": revision,
         "private": private,
-        "8bit_eval": is_8_bit_eval,
-        "is_delta_weight": is_delta_weight,
+        "precision": precision,
+        "weight_type": weight_type,
         "status": "PENDING",
         "submitted_time": current_time,
     }
@@ -217,7 +218,7 @@ def add_new_eval(
 
     OUT_DIR = f"{EVAL_REQUESTS_PATH}/{user_name}"
     os.makedirs(OUT_DIR, exist_ok=True)
-    out_path = f"{OUT_DIR}/{model_path}_eval_request_{private}_{is_8_bit_eval}_{is_delta_weight}.json"
+    out_path = f"{OUT_DIR}/{model_path}_eval_request_{private}_{precision}_{weight_type}.json"
 
     # Check for duplicate submission
     if out_path.split("eval-queue/")[1].lower() in requested_models:
@@ -381,17 +382,29 @@ with demo:
                     revision_name_textbox = gr.Textbox(
                         label="revision", placeholder="main"
                     )
+                    private = gr.Checkbox(
+                        False, label="Private", visible=not IS_PUBLIC
+                    )
 
                 with gr.Column():
-                    is_8bit_toggle = gr.Checkbox(
-                        False, label="8 bit eval", visible=not IS_PUBLIC
+                    precision = gr.Dropdown(
+                        choices=["float16", "bfloat16", "8bit (LLM.int8)", "4bit (QLoRA / FP4)"], 
+                        label="Precision", 
+                        multiselect=False,
+                        value="float16",
+                        max_choices=1,
+                        interactive=True,
                     )
-                    private = gr.Checkbox(
-                        False, label="Private", visible=not IS_PUBLIC
+                    weight_type = gr.Dropdown(
+                        choices=["Original", "Delta", "Adapter"],
+                        label="Weights type", 
+                        multiselect=False,
+                        value="Original",
+                        max_choices=1,
+                        interactive=True,
                     )
-                    is_delta_weight = gr.Checkbox(False, label="Delta weights")
                     base_model_name_textbox = gr.Textbox(
-                        label="base model (for delta)"
+                        label="Base model (for delta or adapter weights)"
                     )
 
             submit_button = gr.Button("Submit Eval")
@@ -402,9 +415,9 @@ with demo:
                     model_name_textbox,
                     base_model_name_textbox,
                     revision_name_textbox,
-                    is_8bit_toggle,
+                    precision,
                     private,
-                    is_delta_weight,
+                    weight_type,
                 ],
                 submission_result,
             )

src/assets/hardcoded_evals.py

@@ -3,7 +3,7 @@ from src.utils_display import AutoEvalColumn, model_hyperlink
 gpt4_values = {
     AutoEvalColumn.model.name: model_hyperlink("https://arxiv.org/abs/2303.08774", "gpt4"),
     AutoEvalColumn.revision.name: "tech report",
-    AutoEvalColumn.is_8bit.name: None,
+    AutoEvalColumn.precision.name: None,
     AutoEvalColumn.average.name: 84.3,
     AutoEvalColumn.arc.name: 96.3,
     AutoEvalColumn.hellaswag.name:  95.3,
@@ -15,7 +15,7 @@ gpt4_values = {
 gpt35_values = {
     AutoEvalColumn.model.name: model_hyperlink("https://arxiv.org/abs/2303.08774", "gpt3.5"),
     AutoEvalColumn.revision.name: "tech report",
-    AutoEvalColumn.is_8bit.name: None,
+    AutoEvalColumn.precision.name: None,
     AutoEvalColumn.average.name: 71.9,
     AutoEvalColumn.arc.name: 85.2,
     AutoEvalColumn.hellaswag.name:  85.5,
@@ -27,7 +27,7 @@ gpt35_values = {
 baseline = {
     AutoEvalColumn.model.name: "<p>Baseline</p>",
     AutoEvalColumn.revision.name: "N/A",
-    AutoEvalColumn.is_8bit.name: None,
+    AutoEvalColumn.precision.name: None,
     AutoEvalColumn.average.name: 25.0,
     AutoEvalColumn.arc.name: 25.0,
     AutoEvalColumn.hellaswag.name:  25.0,

src/assets/text_content.py

@@ -122,12 +122,16 @@ The tasks and few shots parameters are:
 - TruthfulQA: 0-shot, *truthfulqa-mc* (`mc2`)
 - MMLU: 5-shot, *hendrycksTest-abstract_algebra,hendrycksTest-anatomy,hendrycksTest-astronomy,hendrycksTest-business_ethics,hendrycksTest-clinical_knowledge,hendrycksTest-college_biology,hendrycksTest-college_chemistry,hendrycksTest-college_computer_science,hendrycksTest-college_mathematics,hendrycksTest-college_medicine,hendrycksTest-college_physics,hendrycksTest-computer_security,hendrycksTest-conceptual_physics,hendrycksTest-econometrics,hendrycksTest-electrical_engineering,hendrycksTest-elementary_mathematics,hendrycksTest-formal_logic,hendrycksTest-global_facts,hendrycksTest-high_school_biology,hendrycksTest-high_school_chemistry,hendrycksTest-high_school_computer_science,hendrycksTest-high_school_european_history,hendrycksTest-high_school_geography,hendrycksTest-high_school_government_and_politics,hendrycksTest-high_school_macroeconomics,hendrycksTest-high_school_mathematics,hendrycksTest-high_school_microeconomics,hendrycksTest-high_school_physics,hendrycksTest-high_school_psychology,hendrycksTest-high_school_statistics,hendrycksTest-high_school_us_history,hendrycksTest-high_school_world_history,hendrycksTest-human_aging,hendrycksTest-human_sexuality,hendrycksTest-international_law,hendrycksTest-jurisprudence,hendrycksTest-logical_fallacies,hendrycksTest-machine_learning,hendrycksTest-management,hendrycksTest-marketing,hendrycksTest-medical_genetics,hendrycksTest-miscellaneous,hendrycksTest-moral_disputes,hendrycksTest-moral_scenarios,hendrycksTest-nutrition,hendrycksTest-philosophy,hendrycksTest-prehistory,hendrycksTest-professional_accounting,hendrycksTest-professional_law,hendrycksTest-professional_medicine,hendrycksTest-professional_psychology,hendrycksTest-public_relations,hendrycksTest-security_studies,hendrycksTest-sociology,hendrycksTest-us_foreign_policy,hendrycksTest-virology,hendrycksTest-world_religions* (`acc` of `all`)
 
+### Quantization
+To get more information about quantization, see:
+- 8 bits: [blog post](https://huggingface.co/blog/hf-bitsandbytes-integration), [paper](https://arxiv.org/abs/2208.07339)
+- 4 bits: [blog post](https://huggingface.co/blog/4bit-transformers-bitsandbytes), [paper](https://arxiv.org/abs/2305.14314)
+
 # In case of model failure
 If your model is displayed in the `FAILED` category, its execution stopped. 
 Make sure you have followed the above steps first. 
 If everything is done, check you can launch the EleutherAIHarness on your model locally, using the above command without modifications (you can add `--limit` to limit the number of examples per task).
 
-
 """
 
 EVALUATION_QUEUE_TEXT = f"""

src/auto_leaderboard/get_model_metadata.py

@@ -36,7 +36,7 @@ def get_model_license(model_info):
 def get_model_likes(model_info):
     return model_info.likes
 
-size_pattern = re.compile(r"\d+(b|m)")
+size_pattern = re.compile(r"(\d\.)?\d+(b|m)")
 
 def get_model_size(model_name, model_info):
     # In billions
@@ -46,7 +46,7 @@ def get_model_size(model_name, model_info):
         try:
             size_match = re.search(size_pattern, model_name.lower())
             size = size_match.group(0)
-            return round(int(size[:-1]) if size[-1] == "b" else int(size[:-1]) / 1e3, 3)
+            return round(float(size[:-1]) if size[-1] == "b" else float(size[:-1]) / 1e3, 3)
         except AttributeError:
             return None
 

src/auto_leaderboard/load_results.py

@@ -24,7 +24,7 @@ class EvalResult:
     model: str
     revision: str
     results: dict
-    is_8bit: bool = False
+    precision: str = "16bit"
 
     def to_dict(self):
         if self.org is not None:
@@ -34,7 +34,7 @@ class EvalResult:
         data_dict = {}
 
         data_dict["eval_name"] = self.eval_name # not a column, just a save name
-        data_dict[AutoEvalColumn.is_8bit.name] = self.is_8bit
+        data_dict[AutoEvalColumn.precision.name] = self.precision
         data_dict[AutoEvalColumn.model.name] = make_clickable_model(base_model)
         data_dict[AutoEvalColumn.dummy.name] = base_model
         data_dict[AutoEvalColumn.revision.name] = self.revision

src/auto_leaderboard/model_metadata_type.py

@@ -161,3 +161,12 @@ TYPE_METADATA: Dict[str, ModelType] = {
 def get_model_type(leaderboard_data: List[dict]):
     for model_data in leaderboard_data:
         model_data["Type"] = TYPE_METADATA.get(model_data["model_name_for_query"], "N/A")
+        if model_data["Type"] == "N/A":
+            if any([i in model_data["model_name_for_query"] for i in ["finetuned", "-ft-"]]):
+                model_data["Type"] = ModelType.SFT
+            elif any([i in model_data["model_name_for_query"] for i in ["pretrained"]]):
+                model_data["Type"] = ModelType.PT
+            elif any([i in model_data["model_name_for_query"] for i in ["-rl-", "-rlhf-"]]):
+                model_data["Type"] = ModelType.RL
+ 
+ 

src/utils_display.py

@@ -20,8 +20,8 @@ class AutoEvalColumn: # Auto evals column
     hellaswag = ColumnContent("HellaSwag ⬆️", "number", True)
     mmlu = ColumnContent("MMLU ⬆️", "number", True)
     truthfulqa = ColumnContent("TruthfulQA (MC) ⬆️", "number", True)
-    model_type = ColumnContent("Type", "bool", False)
-    is_8bit = ColumnContent("8bit", "bool", False, True)
+    model_type = ColumnContent("Type", "str", False)
+    precision = ColumnContent("Precision", "str", False, True)
     license = ColumnContent("Hub License", "str", False)
     params = ColumnContent("#Params (B)", "number", False)
     likes = ColumnContent("Hub ❤️", "number", False)
@@ -42,8 +42,8 @@ class EvalQueueColumn: # Queue column
     model = ColumnContent("model", "markdown", True)
     revision = ColumnContent("revision", "str", True)
     private = ColumnContent("private", "bool", True)
-    is_8bit = ColumnContent("8bit_eval", "bool", True)
-    has_delta_weight = ColumnContent("is_delta_weight", "bool", True)
+    precision = ColumnContent("precision", "bool", True)
+    weight_type = ColumnContent("weight_type", "str", "Original")
     status = ColumnContent("status", "str", True)
 
 LLAMAS = ["huggingface/llama-7b", "huggingface/llama-13b", "huggingface/llama-30b", "huggingface/llama-65b"]