Create app.py

app.py

@@ -0,0 +1,249 @@
+import pandas as pd
+import streamlit as st
+from huggingface_hub import HfApi
+from utils import ascending_metrics, metric_ranges, CV11_LANGUAGES, FLEURS_LANGUAGES
+import numpy as np
+from st_aggrid import AgGrid, GridOptionsBuilder, JsCode
+from os.path import exists
+import threading
+
+st.set_page_config(layout="wide")
+
+
+def get_model_infos():
+    api = HfApi()
+    model_infos = api.list_models(filter="model-index", cardData=True)
+    return model_infos
+
+
+def parse_metric_value(value):
+    if isinstance(value, str):
+        "".join(value.split("%"))
+        try:
+            value = float(value)
+        except:  # noqa: E722
+            value = None
+    elif isinstance(value, list):
+        if len(value) > 0:
+            value = value[0]
+        else:
+            value = None
+    value = round(value, 4) if isinstance(value, float) else None
+    return value
+
+
+def parse_metrics_rows(meta, only_verified=False):
+    if not isinstance(meta["model-index"], list) or len(meta["model-index"]) == 0 or "results" not in meta["model-index"][0]:
+        return None
+    for result in meta["model-index"][0]["results"]:
+        if not isinstance(result, dict) or "dataset" not in result or "metrics" not in result or "type" not in result["dataset"]:
+            continue
+        dataset = result["dataset"]["type"]
+        if dataset == "":
+            continue
+        row = {"dataset": dataset, "split": "-unspecified-", "config": "-unspecified-"}
+        if "split" in result["dataset"]:
+            row["split"] = result["dataset"]["split"]
+        if "config" in result["dataset"]:
+            row["config"] = result["dataset"]["config"]
+        no_results = True
+        incorrect_results = False
+        for metric in result["metrics"]:
+            name = metric["type"].lower().strip()
+
+            if name in ("model_id", "dataset", "split", "config", "pipeline_tag", "only_verified"):
+                # Metrics are not allowed to be named "dataset", "split", "config", "pipeline_tag"
+                continue
+            value = parse_metric_value(metric.get("value", None))
+            if value is None:
+                continue
+            if name in row:
+                new_metric_better = value < row[name] if name in ascending_metrics else value > row[name]
+            if name not in row or new_metric_better:
+                # overwrite the metric if the new value is better.
+
+                if only_verified:
+                    if "verified" in metric and metric["verified"]:
+                        no_results = False
+                        row[name] = value
+                        if name in metric_ranges:
+                            if value < metric_ranges[name][0] or value > metric_ranges[name][1]:
+                                incorrect_results = True
+                else:
+                    no_results = False
+                    row[name] = value
+                    if name in metric_ranges:
+                        if value < metric_ranges[name][0] or value > metric_ranges[name][1]:
+                            incorrect_results = True
+        if no_results or incorrect_results:
+            continue
+        yield row
+
+
+@st.cache(ttl=0)
+def get_data_wrapper():
+    def get_data(dataframe=None, verified_dataframe=None):
+        data = []
+        verified_data = []
+        print("getting model infos")
+        model_infos = get_model_infos()
+        print("got model infos")
+        for model_info in model_infos:
+            meta = model_info.cardData
+            if meta is None:
+                continue
+            for row in parse_metrics_rows(meta):
+                if row is None:
+                    continue
+                row["model_id"] = model_info.id
+                row["pipeline_tag"] = model_info.pipeline_tag
+                row["only_verified"] = False
+                data.append(row)
+            for row in parse_metrics_rows(meta, only_verified=True):
+                if row is None:
+                    continue
+                row["model_id"] = model_info.id
+                row["pipeline_tag"] = model_info.pipeline_tag
+                row["only_verified"] = True
+                data.append(row)
+        dataframe = pd.DataFrame.from_records(data)
+        dataframe.to_pickle("cache.pkl")
+
+    if exists("cache.pkl"):
+        # If we have saved the results previously, call an asynchronous process
+        # to fetch the results and update the saved file. Don't make users wait
+        # while we fetch the new results. Instead, display the old results for
+        # now. The new results should be loaded when this method
+        # is called again.
+        dataframe = pd.read_pickle("cache.pkl")
+        t = threading.Thread(name="get_data procs", target=get_data)
+        t.start()
+    else:
+        # We have to make the users wait during the first startup of this app.
+        get_data()
+        dataframe = pd.read_pickle("cache.pkl")
+
+    return dataframe
+
+
+dataframe = get_data_wrapper()
+
+st.markdown("# 🤗 Whisper Event: Final Leaderboard")
+
+# query params are used to refine the browser URL as more options are selected
+query_params = st.experimental_get_query_params()
+if "first_query_params" not in st.session_state:
+    st.session_state.first_query_params = query_params
+first_query_params = st.session_state.first_query_params
+
+# define the scope of the leaderboard
+only_verified_results = False
+task = "automatic-speech-recognition"
+selectable_datasets = ["mozilla-foundation/common_voice_11_0", "google/fleurs"]
+dataset_mapping = {"mozilla-foundation/common_voice_11_0": "Common Voice 11", "google/fleurs": "FLEURS"}  # get a 'pretty' name for our datasets
+split = "test"
+selectable_metrics = ["wer", "cer"]
+default_metric = selectable_metrics[0]
+
+# select dataset from list provided
+dataset = st.sidebar.selectbox(
+    "Dataset",
+    selectable_datasets,
+    help="Select a dataset to see the leaderboard!"
+)
+dataset_name = dataset_mapping[dataset]
+
+# slice dataframe to entries of interest
+dataframe = dataframe[dataframe.only_verified == only_verified_results]
+dataset_df = dataframe[dataframe.dataset == dataset]
+dataset_df = dataset_df[dataset_df.split == split]  # hardcoded to "test"
+dataset_df = dataset_df.dropna(axis="columns", how="all")
+
+# get potential dataset configs (languages)
+selectable_configs = list(set(dataset_df["config"]))
+selectable_configs.sort(key=lambda name: name.lower())
+
+if "-unspecified-" in selectable_configs:
+    selectable_configs.remove("-unspecified-")
+
+if dataset == "mozilla-foundation/common_voice_11_0":
+    selectable_configs = [config for config in selectable_configs if config in CV11_LANGUAGES]
+    visual_configs = [f"{config}: {CV11_LANGUAGES[config]}" for config in selectable_configs]
+elif dataset == "google/fleurs":
+    selectable_configs = [config for config in selectable_configs if config in FLEURS_LANGUAGES]
+    visual_configs = [f"{config}: {FLEURS_LANGUAGES[config]}" for config in selectable_configs]
+
+config = st.sidebar.selectbox(
+    "Language",
+    visual_configs,
+    help="Filter the results on the current leaderboard by language."
+)
+config, language = config.split(":")
+
+# just for show -> we've fixed the split to "test"
+split = st.sidebar.selectbox(
+    "Split",
+    [split],
+    index=0,
+    help="View the results for the `test` split for evaluation performance.",
+)
+
+# update browser URL with selections
+current_query_params = {"dataset": [dataset], "config": [config], "split": split}
+st.experimental_set_query_params(**current_query_params)
+
+dataset_df = dataset_df[dataset_df.config == config]
+
+dataset_df = dataset_df.filter(["model_id"] + (["dataset"] if dataset == "-any-" else []) + selectable_metrics)
+dataset_df = dataset_df.dropna(thresh=2)  # Want at least two non-na values (one for model_id and one for a metric).
+
+sorting_metric = st.sidebar.radio(
+    "Sorting Metric",
+    selectable_metrics,
+    index=selectable_metrics.index(default_metric) if default_metric in selectable_metrics else 0,
+    help="Select the metric to sort the leaderboard by. Click on the metric name in the leaderboard to reverse the sorting order."
+)
+
+st.markdown(
+    f"This is the leaderboard for {dataset_name} {language} ({config})."
+)
+
+st.markdown(
+    "Please click on the model's name to be redirected to its model card."
+)
+
+st.markdown(
+    "Want to beat the leaderboard? Don't see your model here? Ensure..."
+)
+
+# Make the default metric appear right after model names and dataset names
+cols = dataset_df.columns.tolist()
+cols.remove(sorting_metric)
+sorting_metric_index = 1 if dataset != "-any-" else 2
+cols = cols[:sorting_metric_index] + [sorting_metric] + cols[sorting_metric_index:]
+dataset_df = dataset_df[cols]
+
+# Sort the leaderboard, giving the sorting metric highest priority and then ordering by other metrics in the case of equal values.
+dataset_df = dataset_df.sort_values(by=cols[sorting_metric_index:], ascending=[metric in ascending_metrics for metric in cols[sorting_metric_index:]])
+dataset_df = dataset_df.replace(np.nan, '-')
+
+# Make the leaderboard
+gb = GridOptionsBuilder.from_dataframe(dataset_df)
+gb.configure_default_column(sortable=False)
+gb.configure_column(
+    "model_id",
+    cellRenderer=JsCode('''function(params) {return '<a target="_blank" href="https://huggingface.co/'+params.value+'">'+params.value+'</a>'}'''),
+)
+
+for name in selectable_metrics:
+    gb.configure_column(name, type=["numericColumn", "numberColumnFilter", "customNumericFormat"], precision=2, aggFunc='sum')
+
+gb.configure_column(
+    sorting_metric,
+    sortable=True,
+    cellStyle=JsCode('''function(params) { return {'backgroundColor': '#FFD21E'}}''')
+)
+
+go = gb.build()
+fit_columns = len(dataset_df.columns) < 10
+AgGrid(dataset_df, gridOptions=go, height=28*len(dataset_df) + (35 if fit_columns else 41), allow_unsafe_jscode=True, fit_columns_on_grid_load=fit_columns, enable_enterprise_modules=False)