v0.0.2 added model details, proper ranking and modal for confirmation

.gitattributes

@@ -8,6 +8,7 @@
 *.h5 filter=lfs diff=lfs merge=lfs -text
 *.joblib filter=lfs diff=lfs merge=lfs -text
 *.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.lz4 filter=lfs diff=lfs merge=lfs -text
 *.mlmodel filter=lfs diff=lfs merge=lfs -text
 *.model filter=lfs diff=lfs merge=lfs -text
 *.msgpack filter=lfs diff=lfs merge=lfs -text
@@ -33,3 +34,22 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+# Audio files - uncompressed
+*.pcm filter=lfs diff=lfs merge=lfs -text
+*.sam filter=lfs diff=lfs merge=lfs -text
+*.raw filter=lfs diff=lfs merge=lfs -text
+# Audio files - compressed
+*.aac filter=lfs diff=lfs merge=lfs -text
+*.flac filter=lfs diff=lfs merge=lfs -text
+*.mp3 filter=lfs diff=lfs merge=lfs -text
+*.ogg filter=lfs diff=lfs merge=lfs -text
+*.wav filter=lfs diff=lfs merge=lfs -text
+# Image files - uncompressed
+*.bmp filter=lfs diff=lfs merge=lfs -text
+*.gif filter=lfs diff=lfs merge=lfs -text
+*.png filter=lfs diff=lfs merge=lfs -text
+*.tiff filter=lfs diff=lfs merge=lfs -text
+# Image files - compressed
+*.jpg filter=lfs diff=lfs merge=lfs -text
+*.jpeg filter=lfs diff=lfs merge=lfs -text
+*.webp filter=lfs diff=lfs merge=lfs -text

app.py

@@ -3,8 +3,10 @@ import os
 import gradio as gr
 import pandas as pd
 from gradio.themes.utils.sizes import text_md
-
-from content import (HEADER_MARKDOWN, LEADERBOARD_TAB_TITLE_MARKDOWN, SUBMISSION_TAB_TITLE_MARKDOWN)
+from gradio_modal import Modal
+from content import (HEADER_MARKDOWN, LEADERBOARD_TAB_TITLE_MARKDOWN, SUBMISSION_TAB_TITLE_MARKDOWN,
+                     MODAL_SUBMIT_MARKDOWN,
+                     SUBMISSION_DETAILS_MARKDOWN, RANKING_AFTER_SUBMISSION_MARKDOWN, MORE_DETAILS_MARKDOWN)
 from server import LeaderboardServer
 
 leaderboard_server = LeaderboardServer()
@@ -48,14 +50,28 @@ def submit_results():
     return gr.update(value='Pre-submit model', visible=True, interactive=True), gr.update(
         visible=False), gr.update(visible=False), gr.update(visible=False), gr.update(
         visible=False), gr.update(visible=False), gr.DataFrame(
-        value=leaderboard_server.get_leaderboard(), visible=True)
+        value=leaderboard_server.get_leaderboard(), visible=True), gr.update(visible=False)
 
 
-def erase_presubmit():
+def erase_pre_submit():
     leaderboard_server.pre_submit = None
     return gr.update(value='Pre-submit model', visible=True, interactive=True), gr.update(
         visible=False), gr.update(visible=False), gr.update(visible=False), gr.update(
-        visible=False), gr.update(visible=False)
+        visible=False), gr.update(visible=False), gr.update(visible=False)
+
+
+def fetch_model_detail(submission_id):
+    metadata = leaderboard_server.get_model_detail(submission_id)
+    return gr.update(value=metadata['description'], visible=True), gr.update(value=metadata['link_to_model'],
+                                                                             visible=True)
+
+
+def show_modal():
+    return gr.update(visible=True)
+
+
+def hide_modal():
+    return gr.update(visible=False)
 
 
 with (gr.Blocks(theme=gr.themes.Soft(text_size=text_md), css="footer {visibility: hidden}") as main):
@@ -71,6 +87,19 @@ with (gr.Blocks(theme=gr.themes.Soft(text_size=text_md), css="footer {visibility
             results_table = gr.DataFrame(leaderboard_server.get_leaderboard(), interactive=False, label=None,
                                          visible=True)
 
+            gr.Markdown(MORE_DETAILS_MARKDOWN)
+            detail_dropdown = gr.Dropdown(choices=leaderboard_server.submission_ids, label="Select model",
+                                          interactive=True)
+
+            with gr.Row():
+                model_description = gr.Text(value='', label='Model description', visible=False, interactive=False)
+                model_url = gr.Text(value='', label='Model url', visible=False, interactive=False)
+
+            detail_dropdown.change(
+                fn=fetch_model_detail,
+                inputs=[detail_dropdown],
+                outputs=[model_description, model_url])
+
         with gr.Tab('Submission'):
             with gr.Column():
                 gr.Markdown(SUBMISSION_TAB_TITLE_MARKDOWN)
@@ -86,16 +115,12 @@ with (gr.Blocks(theme=gr.themes.Soft(text_size=text_md), css="footer {visibility
                 pre_submission_btn = gr.Button(value='Pre-submit model', interactive=True)
 
                 submit_prompt = gr.Markdown(
-                    """
-                    Do you really want to submit a model? This action is irreversible.
-                    """,
+                    SUBMISSION_DETAILS_MARKDOWN,
                     visible=False
                 )
 
                 pre_submit_info = gr.Markdown(
-                    """
-                    This is how will ranking look like after your submission:
-                    """,
+                    RANKING_AFTER_SUBMISSION_MARKDOWN,
                     visible=False
                 )
 
@@ -104,6 +129,11 @@ with (gr.Blocks(theme=gr.themes.Soft(text_size=text_md), css="footer {visibility
                 submission_btn_yes = gr.Button(value='Submit model', interactive=False, visible=False)
                 submission_btn_no = gr.Button(value='Reverse process', interactive=False, visible=False)
 
+                with Modal(visible=False) as modal_submit:
+                    gr.Markdown(MODAL_SUBMIT_MARKDOWN)
+                    modal_submit_yes = gr.Button("Yes", interactive=True)
+                    modal_submit_no = gr.Button("No", interactive=True)
+
                 pre_submission_btn.click(
                     fn=on_submit_pressed,
                     outputs=[pre_submission_btn]
@@ -116,12 +146,23 @@ with (gr.Blocks(theme=gr.themes.Soft(text_size=text_md), css="footer {visibility
                 )
 
                 submission_btn_yes.click(
+                    fn=show_modal,
+                    outputs=[modal_submit]
+                )
+
+                modal_submit_yes.click(
                     fn=submit_results,
                     outputs=[pre_submission_btn, submission_btn_yes, submission_btn_no, submit_prompt, pre_submit_info,
-                             pre_submit_table, results_table]
+                             pre_submit_table, results_table, modal_submit]
+                )
+
+                modal_submit_no.click(
+                    fn=hide_modal,
+                    outputs=[modal_submit]
                 )
+
                 submission_btn_no.click(
-                    fn=erase_presubmit,
+                    fn=erase_pre_submit,
                     outputs=[pre_submission_btn, submission_btn_yes, submission_btn_no, submit_prompt, pre_submit_info,
                              pre_submit_table]
                 )

content.py

@@ -9,7 +9,7 @@ HEADER_MARKDOWN = """
 
 LEADERBOARD_TAB_TITLE_MARKDOWN = """
    ## Leaderboard
-   The leaderboard below shows the current ranking of the models...
+   The leaderboard below shows number of significant wins for each model followed by specific metrics...
    
    """
 
@@ -26,3 +26,17 @@ SUBMISSION_TAB_TITLE_MARKDOWN = """
     This will run a comparison of your model with the existing leaderboard models. 
     After the tournament is complete, you will be able to submit your model to the leaderboard.
 """
+
+RANKING_AFTER_SUBMISSION_MARKDOWN = """
+                    This is how will ranking look like after your submission:
+                    """
+SUBMISSION_DETAILS_MARKDOWN = """
+                    Do you really want to submit a model? This action is irreversible.
+                    """
+MORE_DETAILS_MARKDOWN = """
+## Model details:
+"""
+
+MODAL_SUBMIT_MARKDOWN = """
+Are you sure you want to submit your model?
+"""

model_compare.py

@@ -1,34 +0,0 @@
-from functools import cmp_to_key
-
-
-class ModelCompare:
-
-    def __init__(self, tasks, ranks: dict = None):
-        self.current_task = None
-        self.ranks = ranks
-        self.tasks = tasks
-
-    def compare_models(self, model_a, model_b):
-        if not self.ranks:
-            raise Exception("Missing model rankings")
-
-        res = self.ranks[model_a][model_b][self.current_task]
-        if res:
-            return 1
-        elif not res:
-            return -1
-        else:
-            return -1
-
-    def get_tasks_ranks(self, ranks: dict) -> dict:
-        """Order models based on the significance improvement"""
-
-        self.ranks = ranks
-
-        tasks_ranks = {}
-
-        models = ranks.keys()
-        for task in self.tasks:
-            self.current_task = task
-            tasks_ranks[task] = sorted(models, key=cmp_to_key(self.compare_models))
-        return tasks_ranks

server.py

@@ -7,8 +7,9 @@ import gradio as gr
 import pandas as pd
 from huggingface_hub import HfApi, snapshot_download
 
-from compare_significance import check_significance
-from model_compare import ModelCompare
+from compare_significance import check_significance, SUPPORTED_METRICS
+
+VISIBLE_METRICS = SUPPORTED_METRICS + ["macro_f1"]
 
 api = HfApi()
 
@@ -27,7 +28,6 @@ class LeaderboardServer:
         self.submisssion_id_to_file = {}  # Map submission ids to file paths
         self.tasks_metadata = json.load(open(TASKS_METADATA_PATH))['tasks']
         self.submission_ids = set()
-        self.comparer = ModelCompare(self.tasks_metadata.keys())
         self.fetch_existing_models()
         self.tournament_results = self.load_tournament_results()
         self.pre_submit = None
@@ -59,14 +59,13 @@ class LeaderboardServer:
             self.submisssion_id_to_file[submission_id] = submission
 
     def get_leaderboard(self, tournament_results=None):
-        rank_based_on = tournament_results if tournament_results else self.tournament_results
+        results = tournament_results if tournament_results else self.tournament_results
 
-        if len(rank_based_on) == 0:
+        if len(results) == 0:
             return pd.DataFrame(columns=['No submissions yet'])
         else:
-            ranks = self.comparer.get_tasks_ranks(rank_based_on)
-            results = []
-            for submission in rank_based_on.keys():
+            processed_results = []
+            for submission in results.keys():
                 path = self.submisssion_id_to_file.get(submission)
                 if path is None:
                     if self.pre_submit and submission == self.pre_submit[1]:
@@ -79,14 +78,27 @@ class LeaderboardServer:
                     raise gr.Error(f"Submission [{submission}] not found")
                 submission_id = data["metadata"]["team_name"] + "_" + data["metadata"]["submission_id"]
 
-                local_results = {task: list(task_ranks).index(submission_id) + 1 for task, task_ranks in ranks.items()}
+                local_results = {}
+                for task in self.tasks_metadata.keys():
+                    local_results[task] = 0
+                    for model in results[submission].keys():
+                        if results[submission][model][task]:
+                            local_results[task] += 1
+                    for metric in VISIBLE_METRICS:
+                        metric_value = data['results'][task].get(metric)
+                        if metric_value is not None:
+                            local_results[task + "_" + metric] = metric_value
+
                 local_results["submission_id"] = submission_id
+
                 if self.pre_submit and submission == self.pre_submit[1]:
-                    results.insert(0, local_results)
+                    processed_results.insert(0, local_results)
                 else:
-                    results.append(local_results)
-            dataframe = pd.DataFrame.from_records(results)
-            df_order = ["submission_id"] + [col for col in dataframe.columns if col != "submission_id"]
+                    processed_results.append(local_results)
+            dataframe = pd.DataFrame.from_records(processed_results)
+            df_order = (["submission_id"] + list(self.tasks_metadata.keys()) +
+                        [col for col in dataframe.columns if
+                         col != "submission_id" and col not in self.tasks_metadata.keys()])
             dataframe = dataframe[df_order]
             dataframe = dataframe.rename(columns={key: value["name"] for key, value in self.tasks_metadata.items()})
             return dataframe
@@ -142,3 +154,10 @@ class LeaderboardServer:
                 repo_type=self.repo_type,
                 token=HF_TOKEN,
             )
+
+    def get_model_detail(self, submission_id):
+        path = self.submisssion_id_to_file.get(submission_id)
+        if path is None:
+            raise gr.Error(f"Submission [{submission_id}] not found")
+        data = json.load(open(path))
+        return data["metadata"]