add login button

README.md

@@ -7,6 +7,13 @@ sdk: gradio
 sdk_version: 4.7.1
 app_file: app.py
 pinned: false
+
+hf_oauth: true
+# optional, default duration is 8 hours/480 minutes. Max duration is 30 days/43200 minutes.
+hf_oauth_expiration_minutes: 480
+# optional, see "Scopes" below. "openid profile" is always included.
+hf_oauth_scopes:
+ - inference-api
 ---
 
 Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app_debug.py

@@ -3,12 +3,12 @@ from os.path import isfile, join
 import html
 
 import gradio as gr
-
-import utils.pipe as pipe
-from utils.io_utils import get_logs_file
+import os
+import pipe
+from io_utils import get_logs_file
 
 LOG_PATH = "./tmp"
-CONFIG_PATH = "./cicd/configs/"
+CONFIG_PATH = "./cicd/configs/submitted/"
 MAX_FILES_NUM = 20
 
 
@@ -69,17 +69,19 @@ def get_queue_status():
 
 
 def get_demo():
+    if not os.path.exists(CONFIG_PATH):
+        os.makedirs(CONFIG_PATH)
     with gr.Row():
         gr.HTML(        
             value=get_queue_status,
             every=5,
         )
-    with gr.Accordion(label="Log Files", open=False):
-        with gr.Row():
-            gr.Files(value=get_log_files, label="Log Files", every=10)
+    with gr.Accordion(label="Log Files", open=True):
         with gr.Row():
             gr.Textbox(
                 value=get_logs_file, every=0.5, lines=10, visible=True, label="Current Log File"
             )
+        with gr.Row():
+            gr.Files(value=get_log_files, label="Log Files", every=10)
     with gr.Accordion(label="Config Files", open=False):
         gr.Files(value=get_config_files, label="Config Files", every=10)

app_leaderboard.py

@@ -88,11 +88,29 @@ def get_demo(leaderboard_tab):
     dataset_ids = get_dataset_ids(records)
 
     column_names = records.columns.tolist()
+    issue_columns = column_names[:11]
+    info_columns = column_names[15:]
     default_columns = ["model_id", "dataset_id", "total_issues", "report_link"]
     default_df = records[default_columns]  # extract columns selected
     types = get_types(default_df)
     display_df = get_display_df(default_df)  # the styled dataframe to display
 
+    with gr.Row():
+        with gr.Column():
+          issue_columns_select = gr.CheckboxGroup(
+              label="Issue Columns",
+              choices=issue_columns,
+              value=[],
+              interactive=True,
+          )
+        with gr.Column():
+          info_columns_select = gr.CheckboxGroup(
+              label="Info Columns",
+              choices=info_columns,
+              value=default_columns,
+              interactive=True,
+          )
+    
     with gr.Row():
         task_select = gr.Dropdown(
             label="Task",
@@ -110,42 +128,35 @@ def get_demo(leaderboard_tab):
             interactive=True,
         )
 
-    with gr.Row():
-        columns_select = gr.CheckboxGroup(
-            label="Show columns",
-            choices=column_names,
-            value=default_columns,
-            interactive=True,
-        )
-
     with gr.Row():
         leaderboard_df = gr.DataFrame(display_df, datatype=types, interactive=False)
 
-    def update_leaderboard_records(model_id, dataset_id, columns, task):
+    def update_leaderboard_records(model_id, dataset_id, issue_columns, info_columns, task):
         global update_time
         if datetime.datetime.now() - update_time < datetime.timedelta(minutes=10):
             return gr.update()
         update_time = datetime.datetime.now()
         logger.info("Updating leaderboard records")
         leaderboard.records = get_records_from_dataset_repo(leaderboard.LEADERBOARD)
-        return filter_table(model_id, dataset_id, columns, task)
+        return filter_table(model_id, dataset_id, issue_columns, info_columns, task)
 
     leaderboard_tab.select(
         fn=update_leaderboard_records, 
-        inputs=[model_select, dataset_select, columns_select, task_select], 
+        inputs=[model_select, dataset_select, issue_columns_select, info_columns_select, task_select], 
         outputs=[leaderboard_df])
 
     @gr.on(
         triggers=[
             model_select.change,
             dataset_select.change,
-            columns_select.change,
+            issue_columns_select.change,
+            info_columns_select.change,
             task_select.change,
         ],
-        inputs=[model_select, dataset_select, columns_select, task_select],
+        inputs=[model_select, dataset_select, issue_columns_select, info_columns_select, task_select],
         outputs=[leaderboard_df],
     )
-    def filter_table(model_id, dataset_id, columns, task):
+    def filter_table(model_id, dataset_id, issue_columns, info_columns, task):
         logger.info("Filtering leaderboard records")
         records = leaderboard.records
         # filter the table based on task
@@ -156,8 +167,9 @@ def get_demo(leaderboard_tab):
         if dataset_id and dataset_id != "Any":
             df = df[(df["dataset_id"] == dataset_id)]
 
-        # filter the table based on the columns
-        df = df[columns]
+        # filter the table based on the columns 
+        issue_columns.sort()
+        df = df[info_columns + issue_columns]
         types = get_types(df)
         display_df = get_display_df(df)
         return gr.update(value=display_df, datatype=types, interactive=False)

app_legacy.py

@@ -376,7 +376,7 @@ def get_demo():
         selected = read_scanners("./config.yaml")
         scan_config = selected + ["data_leakage"]
         scanners = gr.CheckboxGroup(
-            choices=scan_config, value=selected, label="Scan Settings", visible=True
+            choices=scan_config, value=selected, visible=True
         )
 
     with gr.Row():

app_text_classification.py

@@ -6,6 +6,7 @@ from utils.io_utils import read_scanners, write_scanners
 from utils.ui_helpers import (
     get_related_datasets_from_leaderboard,
     align_columns_and_show_prediction,
+    get_dataset_splits,
     check_dataset,
     show_hf_token_info,
     precheck_model_ds_enable_example_btn,
@@ -16,12 +17,11 @@ from utils.ui_helpers import (
 )
 
 import logging
-from utils.wordings import (
+from wordings import (
   CONFIRM_MAPPING_DETAILS_MD, 
   INTRODUCTION_MD, 
-  USE_INFERENCE_API_TIP, 
+  LOG_IN_TIPS, 
   CHECK_LOG_SECTION_RAW,
-  HF_TOKEN_INVALID_STYLED
 )
 
 MAX_LABELS = 40
@@ -34,6 +34,8 @@ logger = logging.getLogger(__name__)
 def get_demo():
     with gr.Row():
         gr.Markdown(INTRODUCTION_MD)
+        gr.HTML(LOG_IN_TIPS)
+        gr.LoginButton()
         uid_label = gr.Textbox(
             label="Evaluation ID:", value=uuid.uuid4, visible=False, interactive=False
         )
@@ -58,7 +60,7 @@ def get_demo():
     with gr.Row():
         first_line_ds = gr.DataFrame(label="Dataset Preview", visible=False)
     with gr.Row():
-        loading_status = gr.HTML(visible=True)
+        loading_dataset_info = gr.HTML(visible=True)
     with gr.Row():
         example_btn = gr.Button(
             "Validate Model & Dataset",
@@ -66,11 +68,13 @@ def get_demo():
             variant="primary",
             interactive=False,
         )
-
     with gr.Row():
-        example_input = gr.HTML(visible=False)
+        loading_validation = gr.HTML(visible=True)
+    with gr.Row():
+        validation_result = gr.HTML(visible=False)
     with gr.Row():
-        example_prediction = gr.Label(label="Model Prediction Sample", visible=False)
+        example_input = gr.Textbox(label="Example Input", visible=False, interactive=False)
+        example_prediction = gr.Label(label="Model Sample Prediction", visible=False)
 
     with gr.Row():
         with gr.Accordion(
@@ -89,27 +93,8 @@ def get_demo():
                     for _ in range(MAX_LABELS, MAX_LABELS + MAX_FEATURES):
                         column_mappings.append(gr.Dropdown(visible=False))
 
-    with gr.Accordion(label="Model Wrap Advance Config", open=True):
-        gr.HTML(USE_INFERENCE_API_TIP)
-
-        run_inference = gr.Checkbox(value=True, label="Run with Inference API")
-        inference_token = gr.Textbox(
-            placeholder="hf-xxxxxxxxxxxxxxxxxxxx",
-            value="",
-            label="HF Token for Inference API",
-            visible=True,
-            interactive=True,
-        )
-        inference_token_info = gr.HTML(value=HF_TOKEN_INVALID_STYLED, visible=False)
-
-        inference_token.change(
-            fn=show_hf_token_info,
-            inputs=[inference_token],
-            outputs=[inference_token_info],
-        )
-
-    with gr.Accordion(label="Scanner Advance Config (optional)", open=False):
-        scanners = gr.CheckboxGroup(label="Scan Settings", visible=True)
+    with gr.Accordion(label="Scanner Advanced Config (optional)", open=False):
+        scanners = gr.CheckboxGroup(visible=True)
 
         @gr.on(triggers=[uid_label.change], inputs=[uid_label], outputs=[scanners])
         def get_scanners(uid):
@@ -117,7 +102,16 @@ def get_demo():
             # we remove data_leakage from the default scanners
             # Reason: data_leakage barely raises any issues and takes too many requests
             # when using inference API, causing rate limit error
-            scan_config = selected + ["data_leakage"]
+            scan_config = [
+                "ethical_bias", 
+                "text_perturbation", 
+                "robustness",
+                "performance",
+                "underconfidence",
+                "overconfidence",
+                "spurious_correlation",
+                "data_leakage",
+                ]
             return gr.update(
                 choices=scan_config, value=selected, label="Scan Settings", visible=True
             )
@@ -147,16 +141,24 @@ def get_demo():
         inputs=[model_id_input],
         outputs=[dataset_id_input],
     ).then(
-        fn=check_dataset, 
-        inputs=[dataset_id_input], 
-        outputs=[dataset_config_input, dataset_split_input, loading_status]
+        fn=check_dataset,
+        inputs=[dataset_id_input],
+        outputs=[dataset_config_input, dataset_split_input, loading_dataset_info],
     )
     
     gr.on(
-        triggers=[dataset_id_input.change],
+        triggers=[dataset_id_input.input, dataset_id_input.select],
         fn=check_dataset,
         inputs=[dataset_id_input],
-        outputs=[dataset_config_input, dataset_split_input, loading_status]
+        outputs=[dataset_config_input, dataset_split_input, loading_dataset_info]
+    )
+
+    dataset_config_input.change(fn=get_dataset_splits, inputs=[dataset_id_input, dataset_config_input], outputs=[dataset_split_input])
+
+    gr.on(
+        triggers=[model_id_input.change, dataset_id_input.change, dataset_config_input.change],
+        fn=empty_column_mapping,
+        inputs=[uid_label]
     )
 
     gr.on(
@@ -187,6 +189,7 @@ def get_demo():
     gr.on(
         triggers=[
             model_id_input.change,
+            model_id_input.input,
             dataset_id_input.change,
             dataset_config_input.change,
             dataset_split_input.change,
@@ -198,7 +201,13 @@ def get_demo():
             dataset_config_input,
             dataset_split_input,
         ],
-        outputs=[example_btn, first_line_ds, loading_status],
+        outputs=[
+            example_btn, 
+            first_line_ds,
+            validation_result,
+            example_input,
+            example_prediction,
+            column_mapping_accordion,],
     )
 
     gr.on(
@@ -212,15 +221,14 @@ def get_demo():
             dataset_config_input,
             dataset_split_input,
             uid_label,
-            run_inference,
-            inference_token,
         ],
         outputs=[
+            validation_result,
             example_input,
             example_prediction,
             column_mapping_accordion,
             run_btn,
-            loading_status,
+            loading_validation,
             *column_mappings,
         ],
     )
@@ -235,24 +243,26 @@ def get_demo():
             dataset_id_input,
             dataset_config_input,
             dataset_split_input,
-            run_inference,
-            inference_token,
             uid_label,
         ],
-        outputs=[run_btn, logs, uid_label],
+        outputs=[
+            run_btn, 
+            logs, 
+            uid_label,            
+            validation_result,
+            example_input,
+            example_prediction,
+            column_mapping_accordion,
+          ],
     )
 
     gr.on(
         triggers=[
-            run_inference.input,
-            inference_token.input,
             scanners.input,
         ],
         fn=enable_run_btn,
         inputs=[
             uid_label,
-            run_inference, 
-            inference_token, 
             model_id_input, 
             dataset_id_input, 
             dataset_config_input, 
@@ -266,8 +276,6 @@ def get_demo():
         fn=enable_run_btn,
         inputs=[
             uid_label,
-            run_inference, 
-            inference_token, 
             model_id_input, 
             dataset_id_input, 
             dataset_config_input, 

requirements.txt

@@ -4,4 +4,6 @@ hf-transfer
 torch==2.0.1
 transformers
 datasets
+tabulate
+gradio[oauth]
 -e git+https://github.com/Giskard-AI/cicd.git#egg=giskard-cicd

text_classification.py

@@ -0,0 +1,409 @@
+import json
+import logging
+
+import datasets
+import huggingface_hub
+import pandas as pd
+from transformers import pipeline
+import requests
+import os
+from app_env import HF_WRITE_TOKEN
+
+logger = logging.getLogger(__name__)
+AUTH_CHECK_URL = "https://huggingface.co/api/whoami-v2"
+
+logger = logging.getLogger(__file__)
+
+class HuggingFaceInferenceAPIResponse:
+    def __init__(self, message):
+        self.message = message
+
+
+def get_labels_and_features_from_dataset(ds):
+    try:
+        dataset_features = ds.features
+        label_keys = [i for i in dataset_features.keys() if i.startswith("label")]
+        features = [f for f in dataset_features.keys() if not f.startswith("label")]
+
+        if len(label_keys) == 0: # no labels found
+            # return everything for post processing
+            return list(dataset_features.keys()), list(dataset_features.keys()), None
+        
+        labels = None
+        if not isinstance(dataset_features[label_keys[0]], datasets.ClassLabel):
+            if hasattr(dataset_features[label_keys[0]], "feature"):
+                label_feat = dataset_features[label_keys[0]].feature
+                labels = label_feat.names
+            else:
+                labels = ds.unique(label_keys[0])
+        else:
+            labels = dataset_features[label_keys[0]].names
+        return labels, features, label_keys
+    except Exception as e:
+        logging.warning(
+            f"Get Labels/Features Failed for dataset: {e}"
+        )
+        return None, None, None
+
+def check_model_task(model_id):
+    # check if model is valid on huggingface
+    try:
+        task = huggingface_hub.model_info(model_id).pipeline_tag
+        if task is None:
+            return None
+        return task
+    except Exception:
+        return None
+
+def get_model_labels(model_id, example_input):
+    hf_token = os.environ.get(HF_WRITE_TOKEN, default="")
+    payload = {"inputs": example_input, "options": {"use_cache": True}}
+    response = hf_inference_api(model_id, hf_token, payload)
+    if "error" in response:
+        return None
+    return extract_from_response(response, "label")
+
+def extract_from_response(data, key):
+    results = []
+
+    if isinstance(data, dict):
+        res = data.get(key)
+        if res is not None:
+            results.append(res)
+
+        for value in data.values():
+            results.extend(extract_from_response(value, key))
+
+    elif isinstance(data, list):
+        for element in data:
+            results.extend(extract_from_response(element, key))
+
+    return results
+
+def hf_inference_api(model_id, hf_token, payload):
+    hf_inference_api_endpoint = os.environ.get(
+        "HF_INFERENCE_ENDPOINT", default="https://api-inference.huggingface.co"
+    )
+    url = f"{hf_inference_api_endpoint}/models/{model_id}"
+    headers = {"Authorization": f"Bearer {hf_token}"}
+    response = requests.post(url, headers=headers, json=payload)
+
+    if not hasattr(response, "status_code") or response.status_code != 200:
+        logger.warning(f"Request to inference API returns {response}")
+  
+    try:
+        output = response.json()
+        if "error" in output and "Input is too long" in output["error"]:
+          payload.update({"parameters": {"truncation": True, "max_length": 512}})
+          response = requests.post(url, headers=headers, json=payload)
+          if not hasattr(response, "status_code") or response.status_code != 200:
+              logger.warning(f"Request to inference API returns {response}")
+        return response.json()
+    except Exception:                
+        return {"error": response.content}
+    
+def preload_hf_inference_api(model_id):
+    payload = {"inputs": "This is a test", "options": {"use_cache": True, }}
+    hf_token = os.environ.get(HF_WRITE_TOKEN, default="")
+    hf_inference_api(model_id, hf_token, payload)
+
+def check_model_pipeline(model_id):
+    try:
+        task = huggingface_hub.model_info(model_id).pipeline_tag
+    except Exception:
+        return None
+
+    try:
+        ppl = pipeline(task=task, model=model_id)
+
+        return ppl
+    except Exception:
+        return None
+
+
+def text_classificaiton_match_label_case_unsensative(id2label_mapping, label):
+    for model_label in id2label_mapping.keys():
+        if model_label.upper() == label.upper():
+            return model_label, label
+    return None, label
+
+
+def text_classification_map_model_and_dataset_labels(id2label, dataset_features):
+    id2label_mapping = {id2label[k]: None for k in id2label.keys()}
+    dataset_labels = None
+    for feature in dataset_features.values():
+        if not isinstance(feature, datasets.ClassLabel):
+            continue
+        if len(feature.names) != len(id2label_mapping.keys()):
+            continue
+
+        dataset_labels = feature.names
+        # Try to match labels
+        for label in feature.names:
+            if label in id2label_mapping.keys():
+                model_label = label
+            else:
+                # Try to find case unsensative
+                model_label, label = text_classificaiton_match_label_case_unsensative(
+                    id2label_mapping, label
+                )
+            if model_label is not None:
+                id2label_mapping[model_label] = label
+            else:
+                print(f"Label {label} is not found in model labels")
+
+    return id2label_mapping, dataset_labels
+
+
+"""
+params:
+    column_mapping: dict
+    example: {
+        "text": "sentences",
+        "label": {
+            "label0": "LABEL_0",
+            "label1": "LABEL_1"
+        }
+    }
+    ppl: pipeline
+"""
+
+
+def check_column_mapping_keys_validity(column_mapping, ppl):
+    # get the element in all the list elements
+    column_mapping = json.loads(column_mapping)
+    if "data" not in column_mapping.keys():
+        return True
+    user_labels = set([pair[0] for pair in column_mapping["data"]])
+    model_labels = set([pair[1] for pair in column_mapping["data"]])
+
+    id2label = ppl.model.config.id2label
+    original_labels = set(id2label.values())
+
+    return user_labels == model_labels == original_labels
+
+
+"""
+params:
+    column_mapping: dict
+    dataset_features: dict
+    example: {
+        'text': Value(dtype='string', id=None), 
+        'label': ClassLabel(names=['negative', 'neutral', 'positive'], id=None)
+    }
+"""
+
+
+def infer_text_input_column(column_mapping, dataset_features):
+    # Check whether we need to infer the text input column
+    infer_text_input_column = True
+    feature_map_df = None
+
+    if "text" in column_mapping.keys():
+        dataset_text_column = column_mapping["text"]
+        if dataset_text_column in dataset_features.keys():
+            infer_text_input_column = False
+        else:
+            logging.warning(f"Provided {dataset_text_column} is not in Dataset columns")
+
+    if infer_text_input_column:
+        # Try to retrieve one
+        candidates = [
+            f for f in dataset_features if dataset_features[f].dtype == "string"
+        ]
+        feature_map_df = pd.DataFrame(
+            {"Dataset Features": [candidates[0]], "Model Input Features": ["text"]}
+        )
+        if len(candidates) > 0:
+            logging.debug(f"Candidates are {candidates}")
+            column_mapping["text"] = candidates[0]
+
+    return column_mapping, feature_map_df
+
+
+"""
+params:
+    column_mapping: dict
+    id2label_mapping: dict
+    example:
+    id2label_mapping: {
+        'negative': 'negative', 
+        'neutral': 'neutral', 
+        'positive': 'positive'
+        }
+"""
+
+
+def infer_output_label_column(
+    column_mapping, id2label_mapping, id2label, dataset_labels
+):
+    # Check whether we need to infer the output label column
+    if "data" in column_mapping.keys():
+        if isinstance(column_mapping["data"], list):
+            # Use the column mapping passed by user
+            for user_label, model_label in column_mapping["data"]:
+                id2label_mapping[model_label] = user_label
+    elif None in id2label_mapping.values():
+        column_mapping["label"] = {i: None for i in id2label.keys()}
+        return column_mapping, None
+
+    if "data" not in column_mapping.keys():
+        # Column mapping should contain original model labels
+        column_mapping["label"] = {
+            str(i): id2label_mapping[label]
+            for i, label in zip(id2label.keys(), dataset_labels)
+        }
+
+    id2label_df = pd.DataFrame(
+        {
+            "Dataset Labels": dataset_labels,
+            "Model Prediction Labels": [
+                id2label_mapping[label] for label in dataset_labels
+            ],
+        }
+    )
+
+    return column_mapping, id2label_df
+
+
+def check_dataset_features_validity(d_id, config, split):
+    # We assume dataset is ok here
+    ds = datasets.load_dataset(d_id, config, split=split, trust_remote_code=True)
+    try:
+        dataset_features = ds.features
+    except AttributeError:
+        # Dataset does not have features, need to provide everything
+        return None, None
+        # Load dataset as DataFrame
+    df = ds.to_pandas()
+
+    return df, dataset_features
+
+def select_the_first_string_column(ds):
+    for feature in ds.features.keys():
+        if isinstance(ds[0][feature], str):
+            return feature
+    return None
+
+
+def get_example_prediction(model_id, dataset_id, dataset_config, dataset_split, hf_token):
+    # get a sample prediction from the model on the dataset
+    prediction_input = None
+    prediction_result = None
+    try:
+        # Use the first item to test prediction
+        ds = datasets.load_dataset(dataset_id, dataset_config, split=dataset_split, trust_remote_code=True)
+        if "text" not in ds.features.keys():
+            # Dataset does not have text column
+            prediction_input = ds[0][select_the_first_string_column(ds)]
+        else:
+            prediction_input = ds[0]["text"]
+
+        payload = {"inputs": prediction_input, "options": {"use_cache": True}}
+        results = hf_inference_api(model_id, hf_token, payload)
+
+        if isinstance(results, dict) and "error" in results.keys():
+            if "estimated_time" in results.keys():
+                return prediction_input, HuggingFaceInferenceAPIResponse(
+                    f"Estimated time: {int(results['estimated_time'])}s. Please try again later.")
+            return prediction_input, HuggingFaceInferenceAPIResponse(
+                f"Inference Error: {results['error']}.")
+        
+        while isinstance(results, list):
+            if isinstance(results[0], dict):
+                break
+            results = results[0]
+        prediction_result = {
+            f'{result["label"]}': result["score"] for result in results
+        }
+    except Exception as e:
+        # inference api prediction failed, show the error message
+        logger.error(f"Get example prediction failed {e}")
+        return prediction_input, None
+
+    return prediction_input, prediction_result
+
+
+def get_sample_prediction(ppl, df, column_mapping, id2label_mapping):
+    # get a sample prediction from the model on the dataset
+    prediction_input = None
+    prediction_result = None
+    try:
+        # Use the first item to test prediction
+        prediction_input = df.head(1).at[0, column_mapping["text"]]
+        results = ppl({"text": prediction_input}, top_k=None)
+        prediction_result = {
+            f'{result["label"]}': result["score"] for result in results
+        }
+    except Exception:
+        # Pipeline prediction failed, need to provide labels
+        return prediction_input, None
+
+    # Display results in original label and mapped label
+    prediction_result = {
+        f'{result["label"]}(original) - {id2label_mapping[result["label"]]}(mapped)': result[
+            "score"
+        ]
+        for result in results
+    }
+    return prediction_input, prediction_result
+
+
+def text_classification_fix_column_mapping(column_mapping, ppl, d_id, config, split):
+    # load dataset as pd DataFrame
+    # get features column from dataset
+    df, dataset_features = check_dataset_features_validity(d_id, config, split)
+
+    column_mapping, feature_map_df = infer_text_input_column(
+        column_mapping, dataset_features
+    )
+    if feature_map_df is None:
+        # dataset does not have any features
+        return None, None, None, None, None
+
+    # Retrieve all labels
+    id2label = ppl.model.config.id2label
+
+    # Infer labels
+    id2label_mapping, dataset_labels = text_classification_map_model_and_dataset_labels(
+        id2label, dataset_features
+    )
+    column_mapping, id2label_df = infer_output_label_column(
+        column_mapping, id2label_mapping, id2label, dataset_labels
+    )
+    if id2label_df is None:
+        # does not able to infer output label column
+        return column_mapping, None, None, None, feature_map_df
+
+    # Get a sample prediction
+    prediction_input, prediction_result = get_sample_prediction(
+        ppl, df, column_mapping, id2label_mapping
+    )
+    if prediction_result is None:
+        # does not able to get a sample prediction
+        return column_mapping, prediction_input, None, id2label_df, feature_map_df
+
+    return (
+        column_mapping,
+        prediction_input,
+        prediction_result,
+        id2label_df,
+        feature_map_df,
+    )
+
+def strip_model_id_from_url(model_id):
+    if model_id.startswith("https://huggingface.co/"):
+        return "/".join(model_id.split("/")[-2:])
+    return model_id
+
+def check_hf_token_validity(hf_token):
+    if hf_token == "":
+        return False
+    if not isinstance(hf_token, str):
+        return False
+    # use huggingface api to check the token
+    headers = {"Authorization": f"Bearer {hf_token}"}
+    response = requests.get(AUTH_CHECK_URL, headers=headers)
+    if response.status_code != 200:
+        return False
+    return True

utils/io_utils.py

@@ -1,15 +1,25 @@
 import os
-
+import logging
 import yaml
 
 YAML_PATH = "../cicd/configs"
 LOG_FILE = "../temp_log"
 
+logger = logging.getLogger(__name__)
 
 class Dumper(yaml.Dumper):
     def increase_indent(self, flow=False, *args, **kwargs):
         return super().increase_indent(flow=flow, indentless=False)
 
+def get_submitted_yaml_path(uid):
+    if not os.path.exists(f"{YAML_PATH}/submitted"):
+        os.makedirs(f"{YAML_PATH}/submitted")
+    if not os.path.exists(f"{YAML_PATH}/{uid}_config.yaml"):
+        logger.error(f"config.yaml does not exist for {uid}")
+        os.system(f"cp config.yaml {YAML_PATH}/{uid}_config.yaml")
+    if not os.path.exists(f"{YAML_PATH}/submitted/{uid}_config.yaml"):
+        os.system(f"cp {YAML_PATH}/{uid}_config.yaml {YAML_PATH}/submitted/{uid}_config.yaml")
+    return f"{YAML_PATH}/submitted/{uid}_config.yaml"
 
 def get_yaml_path(uid):
     if not os.path.exists(YAML_PATH):
@@ -72,6 +82,8 @@ def read_column_mapping(uid):
         config = yaml.load(f, Loader=yaml.FullLoader)
         if config:
             column_mapping = config.get("column_mapping", dict())
+    if column_mapping is None:
+        column_mapping = {}
     return column_mapping
 
 

utils/run_jobs.py

@@ -17,7 +17,7 @@ from app_env import (
     HF_SPACE_ID,
     HF_WRITE_TOKEN,
 )
-from utils.io_utils import LOG_FILE, get_yaml_path, write_log_to_user_file
+from io_utils import LOG_FILE, get_submitted_yaml_path, write_log_to_user_file
 from isolated_env import prepare_venv
 from utils.leaderboard import LEADERBOARD
 
@@ -50,7 +50,6 @@ def prepare_env_and_get_command(
     d_id,
     config,
     split,
-    inference,
     inference_token,
     uid,
     label_mapping,
@@ -60,10 +59,6 @@ def prepare_env_and_get_command(
     if os.environ.get("SPACE_ID") == "giskardai/giskard-evaluator":
         leaderboard_dataset = LEADERBOARD
 
-    inference_type = "hf_pipeline"
-    if inference and inference_token:
-        inference_type = "hf_inference_api"
-
     executable = "giskard_scanner"
     try:
         # Copy the current requirements (might be changed)
@@ -98,9 +93,9 @@ def prepare_env_and_get_command(
         "--label_mapping",
         json.dumps(label_mapping),
         "--scan_config",
-        get_yaml_path(uid),
+        get_submitted_yaml_path(uid),
         "--inference_type",
-        inference_type,
+        "hf_inference_api",
         "--inference_api_token",
         inference_token,
     ]

utils/ui_helpers.py

@@ -7,10 +7,15 @@ import datasets
 import gradio as gr
 import pandas as pd
 
-import utils.leaderboard as leaderboard
-from utils.io_utils import read_column_mapping, write_column_mapping
-from utils.run_jobs import save_job_to_pipe
-from utils.text_classification import (
+import leaderboard
+from io_utils import (
+  read_column_mapping, 
+  write_column_mapping, 
+  read_scanners, 
+  write_scanners,
+)
+from run_jobs import save_job_to_pipe
+from text_classification import (
     strip_model_id_from_url,
     check_model_task,
     preload_hf_inference_api,
@@ -26,10 +31,11 @@ from utils.wordings import (
     NOT_TEXT_CLASSIFICATION_MODEL_RAW,
     UNMATCHED_MODEL_DATASET_STYLED_ERROR,
     CHECK_LOG_SECTION_RAW,
-    get_styled_input,
+    VALIDATED_MODEL_DATASET_STYLED,
     get_dataset_fetch_error_raw,
 )
 import os
+from app_env import HF_WRITE_TOKEN
 
 MAX_LABELS = 40
 MAX_FEATURES = 20
@@ -47,9 +53,20 @@ def get_related_datasets_from_leaderboard(model_id):
     datasets_unique = list(model_records["dataset_id"].unique())
 
     if len(datasets_unique) == 0:
-        return gr.update(choices=[], value="")
+        return gr.update(choices=[])
     
-    return gr.update(choices=datasets_unique, value="")
+    return gr.update(choices=datasets_unique)
+
+
+logger = logging.getLogger(__file__)
+
+def get_dataset_splits(dataset_id, dataset_config):
+    try:
+        splits = datasets.get_dataset_split_names(dataset_id, dataset_config, trust_remote_code=True)
+        return gr.update(choices=splits, value=splits[0], visible=True)
+    except Exception as e:
+        logger.warn(f"Check your dataset {dataset_id} and config {dataset_config}: {e}")
+        return gr.update(visible=False)
 
 def check_dataset(dataset_id):
     logger.info(f"Loading {dataset_id}")
@@ -61,9 +78,7 @@ def check_dataset(dataset_id):
                 gr.update(visible=False),
                 ""
             )
-        splits = datasets.get_dataset_split_names(
-                        dataset_id, configs[0], trust_remote_code=True
-                    )
+        splits = datasets.get_dataset_split_names(dataset_id, configs[0], trust_remote_code=True)
         return (
             gr.update(choices=configs, value=configs[0], visible=True),
             gr.update(choices=splits, value=splits[0], visible=True),
@@ -125,7 +140,7 @@ def list_labels_and_features_from_dataset(ds_labels, ds_features, model_labels,
         ds_labels = list(shared_labels)
     if len(ds_labels) > MAX_LABELS:
         ds_labels = ds_labels[:MAX_LABELS]
-        gr.Warning(f"The number of labels is truncated to length {MAX_LABELS}")
+        gr.Warning(f"Too many labels to display for this spcae. We do not support more than {MAX_LABELS} in this space. You can use cli tool at https://github.com/Giskard-AI/cicd.")
 
     # sort labels to make sure the order is consistent
     # prediction gives the order based on probability
@@ -166,33 +181,67 @@ def list_labels_and_features_from_dataset(ds_labels, ds_features, model_labels,
 
 def precheck_model_ds_enable_example_btn(
     model_id, dataset_id, dataset_config, dataset_split
-):
-    if model_id == "" or dataset_id == "":
-        return (gr.update(interactive=False), gr.update(visible=False), "")
+):  
     model_id = strip_model_id_from_url(model_id)
     model_task = check_model_task(model_id)
     preload_hf_inference_api(model_id)
-    if model_task is None or model_task != "text-classification":
-        gr.Warning(NOT_TEXT_CLASSIFICATION_MODEL_RAW)
-        return (gr.update(interactive=False), gr.update(visible=False), "")
-    
+
     if dataset_config is None or dataset_split is None or len(dataset_config) == 0:
-        return (gr.update(interactive=False), gr.update(visible=False), "")
-    
+        return (
+            gr.update(interactive=False), 
+            gr.update(visible=False),
+            gr.update(visible=False),
+            gr.update(visible=False),
+            gr.update(visible=False),
+            gr.update(visible=False),
+        )
+
     try:
         ds = datasets.load_dataset(dataset_id, dataset_config, trust_remote_code=True)
         df: pd.DataFrame = ds[dataset_split].to_pandas().head(5)
-        ds_labels, ds_features = get_labels_and_features_from_dataset(ds[dataset_split])
+        ds_labels, ds_features, _ = get_labels_and_features_from_dataset(ds[dataset_split])
+
+        if model_task is None or model_task != "text-classification":
+          gr.Warning(NOT_TEXT_CLASSIFICATION_MODEL_RAW)
+          return (
+              gr.update(interactive=False), 
+              gr.update(value=df, visible=True),
+              gr.update(visible=False),
+              gr.update(visible=False),
+              gr.update(visible=False),
+              gr.update(visible=False),
+          )
 
         if not isinstance(ds_labels, list) or not isinstance(ds_features, list):
             gr.Warning(CHECK_CONFIG_OR_SPLIT_RAW)
-            return (gr.update(interactive=False), gr.update(value=df, visible=True), "")
+            return (
+                gr.update(interactive=False), 
+                gr.update(value=df, visible=True),
+                gr.update(visible=False),
+                gr.update(visible=False),
+                gr.update(visible=False),
+                gr.update(visible=False),
+            )
 
-        return (gr.update(interactive=True), gr.update(value=df, visible=True), "")
+        return (
+                gr.update(interactive=True), 
+                gr.update(value=df, visible=True),
+                gr.update(visible=False),
+                gr.update(visible=False),
+                gr.update(visible=False),
+                gr.update(visible=False),
+            )
     except Exception as e:
         # Config or split wrong
         logger.warn(f"Check your dataset {dataset_id} and config {dataset_config} on split {dataset_split}: {e}")
-        return (gr.update(interactive=False), gr.update(value=pd.DataFrame(), visible=False), "")
+        return (
+            gr.update(interactive=False),
+            gr.update(visible=False),
+            gr.update(visible=False),
+            gr.update(visible=False),
+            gr.update(visible=False),
+            gr.update(visible=False),
+        )
 
 
 def align_columns_and_show_prediction(
@@ -201,8 +250,8 @@ def align_columns_and_show_prediction(
     dataset_config,
     dataset_split,
     uid,
-    run_inference,
-    inference_token,
+    profile: gr.OAuthProfile | None,
+    oauth_token: gr.OAuthToken | None,
 ):
     model_id = strip_model_id_from_url(model_id)
     model_task = check_model_task(model_id)
@@ -221,7 +270,7 @@ def align_columns_and_show_prediction(
         gr.Dropdown(visible=False) for _ in range(MAX_LABELS + MAX_FEATURES)
     ]
     
-    hf_token = os.environ.get("HF_WRITE_TOKEN", default="")
+    hf_token = os.environ.get(HF_WRITE_TOKEN, default="")
 
     prediction_input, prediction_response = get_example_prediction(
         model_id, dataset_id, dataset_config, dataset_split, hf_token
@@ -229,6 +278,7 @@ def align_columns_and_show_prediction(
 
     if prediction_input is None or prediction_response is None:
         return (
+            gr.update(visible=False),
             gr.update(visible=False),
             gr.update(visible=False),
             gr.update(visible=False, open=False),
@@ -239,6 +289,7 @@ def align_columns_and_show_prediction(
 
     if isinstance(prediction_response, HuggingFaceInferenceAPIResponse):
         return (
+            gr.update(visible=False),
             gr.update(visible=False),
             gr.update(visible=False),
             gr.update(visible=False, open=False),
@@ -250,12 +301,13 @@ def align_columns_and_show_prediction(
     model_labels = list(prediction_response.keys())
     
     ds = datasets.load_dataset(dataset_id, dataset_config, split=dataset_split, trust_remote_code=True)
-    ds_labels, ds_features = get_labels_and_features_from_dataset(ds)
+    ds_labels, ds_features, _ = get_labels_and_features_from_dataset(ds)
 
     # when dataset does not have labels or features
     if not isinstance(ds_labels, list) or not isinstance(ds_features, list):
         gr.Warning(CHECK_CONFIG_OR_SPLIT_RAW)
         return (
+            gr.update(visible=False),
             gr.update(visible=False),
             gr.update(visible=False),
             gr.update(visible=False, open=False),
@@ -268,6 +320,7 @@ def align_columns_and_show_prediction(
         return (
             gr.update(value=UNMATCHED_MODEL_DATASET_STYLED_ERROR, visible=True),
             gr.update(visible=False),
+            gr.update(visible=False),
             gr.update(visible=False, open=False),
             gr.update(interactive=False),
             "",
@@ -289,18 +342,20 @@ def align_columns_and_show_prediction(
     ):
         return (
             gr.update(value=MAPPING_STYLED_ERROR_WARNING, visible=True),
-            gr.update(visible=False),
+            gr.update(value=prediction_input, lines=min(len(prediction_input)//225 + 1, 5), visible=True),
+            gr.update(value=prediction_response, visible=True),
             gr.update(visible=True, open=True),
-            gr.update(interactive=(run_inference and inference_token != "")),
+            gr.update(interactive=(profile is not None and oauth_token is not None)),
             "",
             *column_mappings,
         )
 
     return (
-        gr.update(value=get_styled_input(prediction_input), visible=True),
+        gr.update(value=VALIDATED_MODEL_DATASET_STYLED, visible=True),
+        gr.update(value=prediction_input, lines=min(len(prediction_input)//225 + 1, 5), visible=True),
         gr.update(value=prediction_response, visible=True),
         gr.update(visible=True, open=False),
-        gr.update(interactive=(run_inference and inference_token != "")),
+        gr.update(interactive=(profile is not None and oauth_token is not None)),
         "",
         *column_mappings,
     )
@@ -308,18 +363,20 @@ def align_columns_and_show_prediction(
 
 def check_column_mapping_keys_validity(all_mappings):
     if all_mappings is None:
+        logger.warning("all_mapping is None")
         gr.Warning(CONFIRM_MAPPING_DETAILS_FAIL_RAW)
         return False
 
     if "labels" not in all_mappings.keys():
-        gr.Warning(CONFIRM_MAPPING_DETAILS_FAIL_RAW)
+        logger.warning(f"Label mapping is not valid, all_mappings: {all_mappings}")
         return False
 
     return True
 
-def enable_run_btn(uid, run_inference, inference_token, model_id, dataset_id, dataset_config, dataset_split):
-    if not run_inference or inference_token == "":
-        logger.warn("Inference API is not enabled")
+def enable_run_btn(uid, model_id, dataset_id, dataset_config, dataset_split, profile: gr.OAuthProfile | None, oath_token: gr.OAuthToken | None):
+    if profile is None:
+        return gr.update(interactive=False)
+    if oath_token is None:
         return gr.update(interactive=False)
     if model_id == "" or dataset_id == "" or dataset_config == "" or dataset_split == "":
         logger.warn("Model id or dataset id is not selected")
@@ -330,26 +387,27 @@ def enable_run_btn(uid, run_inference, inference_token, model_id, dataset_id, da
         logger.warn("Column mapping is not valid")
         return gr.update(interactive=False)
 
-    if not check_hf_token_validity(inference_token):
-        logger.warn("HF token is not valid")
-        return gr.update(interactive=False)
-    return gr.update(interactive=True)
-
-def construct_label_and_feature_mapping(all_mappings, ds_labels, ds_features):
+def construct_label_and_feature_mapping(all_mappings, ds_labels, ds_features, label_keys=None):
     label_mapping = {}
     if len(all_mappings["labels"].keys()) != len(ds_labels):
-        logger.warn("Label mapping corrupted: " + CONFIRM_MAPPING_DETAILS_FAIL_RAW)
+        logger.warn(f"""Label mapping corrupted: {CONFIRM_MAPPING_DETAILS_FAIL_RAW}. 
+                    \nall_mappings: {all_mappings}\nds_labels: {ds_labels}""")
       
     if len(all_mappings["features"].keys()) != len(ds_features):
-        logger.warn("Feature mapping corrupted: " + CONFIRM_MAPPING_DETAILS_FAIL_RAW)
+        logger.warn(f"""Feature mapping corrupted: {CONFIRM_MAPPING_DETAILS_FAIL_RAW}. 
+                    \nall_mappings: {all_mappings}\nds_features: {ds_features}""")
 
     for i, label in zip(range(len(ds_labels)),  ds_labels):
         # align the saved labels with dataset labels order
         label_mapping.update({str(i): all_mappings["labels"][label]})
 
     if "features" not in all_mappings.keys():
+        logger.warning("features not in all_mappings")
         gr.Warning(CONFIRM_MAPPING_DETAILS_FAIL_RAW)
+
     feature_mapping = all_mappings["features"]
+    if len(label_keys) > 0:
+      feature_mapping.update({"label": label_keys[0]})
     return label_mapping, feature_mapping
 
 def show_hf_token_info(token):
@@ -358,16 +416,18 @@ def show_hf_token_info(token):
         return gr.update(visible=True)
     return gr.update(visible=False)
 
-def try_submit(m_id, d_id, config, split, inference, inference_token, uid):
+def try_submit(m_id, d_id, config, split, uid, profile: gr.OAuthProfile | None, oath_token: gr.OAuthToken | None):
+    print(oath_token.token)
+    print(".>>>>>>>>>>>>>>>>>>>>>>")
     all_mappings = read_column_mapping(uid)
     if not check_column_mapping_keys_validity(all_mappings):
         return (gr.update(interactive=True), gr.update(visible=False))
 
     # get ds labels and features again for alignment
     ds = datasets.load_dataset(d_id, config, split=split, trust_remote_code=True)
-    ds_labels, ds_features = get_labels_and_features_from_dataset(ds)
-    label_mapping, feature_mapping = construct_label_and_feature_mapping(all_mappings, ds_labels, ds_features)
-
+    ds_labels, ds_features, label_keys = get_labels_and_features_from_dataset(ds)
+    label_mapping, feature_mapping = construct_label_and_feature_mapping(all_mappings, ds_labels, ds_features, label_keys)
+    
     eval_str = f"[{m_id}]<{d_id}({config}, {split} set)>"
     save_job_to_pipe(
         uid,
@@ -376,8 +436,7 @@ def try_submit(m_id, d_id, config, split, inference, inference_token, uid):
             d_id,
             config,
             split,
-            inference,
-            inference_token,
+            oath_token.token,
             uid,
             label_mapping,
             feature_mapping,
@@ -387,8 +446,16 @@ def try_submit(m_id, d_id, config, split, inference, inference_token, uid):
     )
     gr.Info("Your evaluation has been submitted")
 
+    new_uid = uuid.uuid4()
+    scanners = read_scanners(uid)
+    write_scanners(scanners, new_uid)
+
     return (
         gr.update(interactive=False),  # Submit button
         gr.update(value=f"{CHECK_LOG_SECTION_RAW}Your job id is: {uid}. ", lines=5, visible=True, interactive=False),
-        uuid.uuid4(),  # Allocate a new uuid
+        new_uid,  # Allocate a new uuid
+        gr.update(visible=False),
+        gr.update(visible=False),
+        gr.update(visible=False),
+        gr.update(visible=False),
     )

utils/wordings.py

@@ -2,23 +2,24 @@ INTRODUCTION_MD = """
                 <h1 style="text-align: center;">
                 🐢Giskard Evaluator - Text Classification
                 </h1>
-                Welcome to the Giskard Evaluator Space! Get a model vulnerability report immediately by simply sharing your model and dataset id below.
+                Welcome to the Giskard Evaluator Space! Get a model vulnerability report immediately by simply sharing your model and dataset id below. 
+                You can also checkout our library documentation <a href="https://docs.giskard.ai/en/latest/getting_started/quickstart/index.html">here</a>.
                 """
 CONFIRM_MAPPING_DETAILS_MD = """
                             <h1 style="text-align: center;">
                             Confirm Pre-processing Details
                             </h1>
-                            Make sure the output variable's labels and the input variable's name are accurately mapped across both the dataset and the model.
+                            Make sure the output variable's labels and the input variable's name are accurately mapped across both the dataset and the model. You can select the output variable's labels from the dropdowns below.
                             """
 CONFIRM_MAPPING_DETAILS_FAIL_MD = """
                             <h1 style="text-align: center;">
                             Confirm Pre-processing Details
                             </h1>
-                            We're unable to automatically map the input variable's name and output variable's labels of your dataset with the model's. <b>Please manually check the mapping below.</b>
+                            We're unable to automatically map the input variable's name and output variable's labels of your dataset with the model's. Please manually check the mapping below.
                             """
 
 CONFIRM_MAPPING_DETAILS_FAIL_RAW = """
-                            We're unable to automatically map the input variable's name and output variable's labels of your dataset with the model's. <b>Please manually check the mapping below.</b>
+                            We're unable to automatically map the input variable's name and output variable's labels of your dataset with the model's. Please manually check the mapping below.
                             """
 
 CHECK_CONFIG_OR_SPLIT_RAW = """
@@ -38,7 +39,7 @@ PREDICTION_SAMPLE_MD = """
 
 MAPPING_STYLED_ERROR_WARNING = """
                         <h3 style="text-align: center;color: orange; background-color: #fff0f3; border-radius: 8px; padding: 10px; ">
-                        ⚠️ We're unable to automatically map the input variable's name and output variable's labels of your dataset with the model's. <b>Please manually check the mapping below.</b>
+                        ⚠️ We're unable to automatically map the input variable's name and output variable's labels of your dataset with the model's. Please manually check the mapping below.
                         </h3>
                         """
 
@@ -57,7 +58,11 @@ USE_INFERENCE_API_TIP = """
                 <a href="https://huggingface.co/docs/api-inference/detailed_parameters#text-classification-task">
                     Hugging Face Inference API
                 </a>
-                . Please input your <a href="https://huggingface.co/settings/tokens">Hugging Face token</a> to do so.
+                . Please input your <a href="https://huggingface.co/settings/tokens">Hugging Face token</a> to do so. You can find it <a href="https://huggingface.co/settings/tokens">here</a>.
+            """
+
+LOG_IN_TIPS = """
+            To use the Hugging Face Inference API, you need to log in to your Hugging Face account.
             """
 
 HF_TOKEN_INVALID_STYLED= """
@@ -66,10 +71,10 @@ HF_TOKEN_INVALID_STYLED= """
                 </p>
                 """
 
+VALIDATED_MODEL_DATASET_STYLED = """
+            <h3 style="text-align: center;color: #4ca154; background-color: #e2fbe8; border-radius: 8px; padding: 10px; ">
+            Your model and dataset have been validated!
+            </h3>"""
+
 def get_dataset_fetch_error_raw(error):
     return f"""Sorry you cannot use this dataset because {error}. Contact HF team to support this dataset."""
-
-def get_styled_input(input):
-    return f"""<h3 style="text-align: center;color: #4ca154; background-color: #e2fbe8; border-radius: 8px; padding: 10px; ">
-            Your model and dataset have been validated! <br /> Sample input: {input}
-            </h3>"""