Added icons for types + fixed pending queue

app.py

@@ -99,7 +99,6 @@ def get_leaderboard_df():
 
 
 def get_evaluation_queue_df():
-    # todo @saylortwift: replace the repo by the one you created for the eval queue
     if eval_queue:
         print("Pulling changes for the evaluation queue.")
         eval_queue.git_pull()
@@ -141,7 +140,7 @@ def get_evaluation_queue_df():
                 data["model"] = make_clickable_model(data["model"])
                 all_evals.append(data)
 
-    pending_list = [e for e in all_evals if e["status"] == "PENDING"]
+    pending_list = [e for e in all_evals if e["status"] in ["PENDING", "RERUN"]]
     running_list = [e for e in all_evals if e["status"] == "RUNNING"]
     finished_list = [e for e in all_evals if e["status"].startswith("FINISHED")]
     df_pending = pd.DataFrame.from_records(pending_list, columns=EVAL_COLS)
@@ -388,6 +387,14 @@ with demo:
                     private = gr.Checkbox(
                         False, label="Private", visible=not IS_PUBLIC
                     )
+                    model_type = gr.Dropdown(
+                        choices=["pretrained", "fine-tuned", "with RL"], 
+                        label="Model type", 
+                        multiselect=False,
+                        value="pretrained",
+                        max_choices=1,
+                        interactive=True,
+                    )
 
                 with gr.Column():
                     precision = gr.Dropdown(
@@ -398,14 +405,6 @@ with demo:
                         max_choices=1,
                         interactive=True,
                     )
-                    model_type = gr.Dropdown(
-                        choices=["pretrained", "fine-tuned", "with RL"], 
-                        label="Model type", 
-                        multiselect=False,
-                        value="pretrained",
-                        max_choices=1,
-                        interactive=True,
-                    )
                     weight_type = gr.Dropdown(
                         choices=["Original", "Delta", "Adapter"],
                         label="Weights type", 

src/assets/hardcoded_evals.py

@@ -10,6 +10,7 @@ gpt4_values = {
     AutoEvalColumn.mmlu.name:  86.4,
     AutoEvalColumn.truthfulqa.name:  59.0,
     AutoEvalColumn.dummy.name: "GPT-4",
+    AutoEvalColumn.model_type.name: "",
 }
 
 gpt35_values = {
@@ -22,6 +23,7 @@ gpt35_values = {
     AutoEvalColumn.mmlu.name:  70.0,
     AutoEvalColumn.truthfulqa.name:  47.0,
     AutoEvalColumn.dummy.name: "GPT-3.5",
+    AutoEvalColumn.model_type.name: "",
 }
 
 baseline = {
@@ -34,5 +36,6 @@ baseline = {
     AutoEvalColumn.mmlu.name:  25.0,
     AutoEvalColumn.truthfulqa.name:  25.0,
     AutoEvalColumn.dummy.name: "baseline",
+    AutoEvalColumn.model_type.name: "",
 }
 

src/assets/text_content.py

@@ -128,6 +128,13 @@ To get more information about quantization, see:
 - 8 bits: [blog post](https://huggingface.co/blog/hf-bitsandbytes-integration), [paper](https://arxiv.org/abs/2208.07339)
 - 4 bits: [blog post](https://huggingface.co/blog/4bit-transformers-bitsandbytes), [paper](https://arxiv.org/abs/2305.14314)
 
+### Icons
+🟢 means that the model is pretrained
+🔶 that it is finetuned
+🟦 that is was trained with RL.
+If there is no icon, we have not uploaded the information on the model yet, feel free to open an issue with the model information!
+
+
 # In case of model failure
 If your model is displayed in the `FAILED` category, its execution stopped. 
 Make sure you have followed the above steps first. 

src/auto_leaderboard/load_results.py

@@ -26,6 +26,8 @@ class EvalResult:
     revision: str
     results: dict
     precision: str = "16bit"
+    model_type: str = ""
+    weight_type: str = ""
 
     def to_dict(self):
         if self.org is not None:
@@ -35,7 +37,9 @@ class EvalResult:
         data_dict = {}
 
         data_dict["eval_name"] = self.eval_name # not a column, just a save name
+        data_dict["weight_type"] = self.weight_type  # not a column, just a save name
         data_dict[AutoEvalColumn.precision.name] = self.precision
+        data_dict[AutoEvalColumn.model_type.name] = self.model_type
         data_dict[AutoEvalColumn.model.name] = make_clickable_model(base_model)
         data_dict[AutoEvalColumn.dummy.name] = base_model
         data_dict[AutoEvalColumn.revision.name] = self.revision
@@ -92,7 +96,7 @@ def parse_eval_result(json_filepath: str) -> Tuple[str, list[dict]]:
             continue
         mean_acc = round(np.mean(accs) * 100.0, 1)
         eval_results.append(EvalResult(
-            result_key, org, model, model_sha, {benchmark: mean_acc}
+            eval_name=result_key, org=org, model=model, revision=model_sha, results={benchmark: mean_acc}, #todo model_type=, weight_type=
         ))
 
     return result_key, eval_results

src/auto_leaderboard/model_metadata_type.py

@@ -2,6 +2,8 @@ from dataclasses import dataclass
 from enum import Enum
 from typing import Dict, List
 
+from ..utils_display import AutoEvalColumn
+
 @dataclass
 class ModelInfo:
     name: str
@@ -167,23 +169,24 @@ TYPE_METADATA: Dict[str, ModelType] = {
 
 def get_model_type(leaderboard_data: List[dict]):
     for model_data in leaderboard_data:
-        # Init
-        model_data["Type name"] = "N/A"
-        model_data["Type"] = ""
-
+        # Todo @clefourrier once requests are connected with results 
+        is_delta = False # (model_data["weight_type"] != "Original")
         # Stored information
         if model_data["model_name_for_query"] in TYPE_METADATA:
-            model_data["Type name"] = TYPE_METADATA[model_data["model_name_for_query"]].value.name
-            model_data["Type"] = TYPE_METADATA[model_data["model_name_for_query"]].value.symbol
-        else: # Supposed from the name
-            if any([i in model_data["model_name_for_query"] for i in ["finetuned", "-ft-"]]):
-                model_data["Type name"] = ModelType.SFT.value.name
-                model_data["Type"] = ModelType.SFT.value.symbol
-            elif any([i in model_data["model_name_for_query"] for i in ["pretrained"]]):
-                model_data["Type name"] = ModelType.PT.value.name
-                model_data["Type"] = ModelType.PT.value.symbol
-            elif any([i in model_data["model_name_for_query"] for i in ["-rl-", "-rlhf-"]]):
-                model_data["Type name"] = ModelType.RL.value.name
-                model_data["Type"] = ModelType.RL.value.symbol
+            model_data[AutoEvalColumn.model_type.name] = TYPE_METADATA[model_data["model_name_for_query"]].value.name
+            model_data[AutoEvalColumn.model_type_symbol.name] = TYPE_METADATA[model_data["model_name_for_query"]].value.symbol + ("🔺" if is_delta else "")
+        # Inferred from the name or the selected type 
+        elif model_data[AutoEvalColumn.model_type.name] == "pretrained" or  any([i in model_data["model_name_for_query"] for i in ["pretrained"]]):
+            model_data[AutoEvalColumn.model_type.name] = ModelType.PT.value.name
+            model_data[AutoEvalColumn.model_type_symbol.name] = ModelType.PT.value.symbol + ("🔺" if is_delta else "")
+        elif model_data[AutoEvalColumn.model_type.name] == "finetuned" or any([i in model_data["model_name_for_query"] for i in ["finetuned", "-ft-"]]):
+            model_data[AutoEvalColumn.model_type.name] = ModelType.SFT.value.name
+            model_data[AutoEvalColumn.model_type_symbol.name] = ModelType.SFT.value.symbol + ("🔺" if is_delta else "")
+        elif model_data[AutoEvalColumn.model_type.name] == "with RL" or any([i in model_data["model_name_for_query"] for i in ["-rl-", "-rlhf-"]]):
+            model_data[AutoEvalColumn.model_type.name] = ModelType.RL.value.name
+            model_data[AutoEvalColumn.model_type_symbol.name] = ModelType.RL.value.symbol + ("🔺" if is_delta else "")
+        else:
+            model_data[AutoEvalColumn.model_type.name] = "N/A"
+            model_data[AutoEvalColumn.model_type_symbol.name] = ("🔺" if is_delta else "")
  
  

src/utils_display.py

@@ -14,14 +14,14 @@ def fields(raw_class):
 
 @dataclass(frozen=True)
 class AutoEvalColumn: # Auto evals column
-    model_type_symbol = ColumnContent("Type", "str", True)
+    model_type_symbol = ColumnContent("T", "str", True)
     model = ColumnContent("Model", "markdown", True)
     average = ColumnContent("Average ⬆️", "number", True)
     arc = ColumnContent("ARC", "number", True)
     hellaswag = ColumnContent("HellaSwag", "number", True)
     mmlu = ColumnContent("MMLU", "number", True)
     truthfulqa = ColumnContent("TruthfulQA (MC) ⬆️", "number", True)
-    model_type = ColumnContent("Type name", "str", False)
+    model_type = ColumnContent("Type", "str", False)
     precision = ColumnContent("Precision", "str", False, True)
     license = ColumnContent("Hub License", "str", False)
     params = ColumnContent("#Params (B)", "number", False)