Refactor the code

app.py

@@ -1,4 +1,8 @@
 from src.view.view import create_interface
 
-demo = create_interface()
-demo.launch()
+global demo
+demo = None
+
+if __name__ == "__main__":
+    demo = create_interface()
+    demo.launch()

src/logic/data_fetching.py

@@ -1,5 +1,7 @@
+from functools import partial
 import os
 import json
+import re
 import tempfile
 from pathlib import Path
 from concurrent.futures import ThreadPoolExecutor
@@ -9,30 +11,25 @@ from datatrove.utils.stats import MetricStatsDict
 import gradio as gr
 import tenacity
 
+from src.logic.graph_settings import Grouping
+
 def find_folders(base_folder: str, path: str) -> List[str]:
-    base_folder = get_datafolder(base_folder)
-    if not base_folder.exists(path):
+    base_folder_df = get_datafolder(base_folder)
+    if not base_folder_df.exists(path):
         return []
     return sorted(
         [
-            folder["name"]
-            for folder in base_folder.ls(path, detail=True)
-            if folder["type"] == "directory" and not folder["name"].rstrip("/") == path
+            folder
+            for folder,info in base_folder_df.find(path, maxdepth=1, withdirs=True, detail=True).items()
+            if info["type"] == "directory" and not (folder.rstrip("/") == path.rstrip("/"))
         ]
     )
 
-def find_metrics_folders(base_folder: str) -> List[str]:
-    base_data_df = get_datafolder(base_folder)
-    dirs = sorted(
-        folder
-        for folder, info in base_data_df.find("", detail=True, maxdepth=1, withdirs=True).items()
-            if info["type"] == "directory"
-        )
-    return sorted(list(set(dirs)))
-
 def fetch_datasets(base_folder: str):
-    datasets = sorted(find_metrics_folders(base_folder))
-    return datasets, gr.update(choices=datasets, value=None), fetch_groups(base_folder, datasets, None, "union")
+    datasets = sorted(find_folders(base_folder, ""))
+    if len(datasets) == 0:
+        raise ValueError("No datasets found")
+    return datasets
 
 def fetch_groups(base_folder: str, datasets: List[str], old_groups: str, type: str = "intersection"):
     if not datasets:
@@ -55,7 +52,7 @@ def fetch_groups(base_folder: str, datasets: List[str], old_groups: str, type: s
     if not value and len(new_choices) == 1:
         value = list(new_choices)[0]
 
-    return gr.update(choices=sorted(list(new_choices)), value=value)
+    return gr.Dropdown(choices=sorted(list(new_choices)), value=value)
 
 def fetch_metrics(base_folder: str, datasets: List[str], group: str, old_metrics: str, type: str = "intersection"):
     if not group:
@@ -79,7 +76,7 @@ def fetch_metrics(base_folder: str, datasets: List[str], group: str, old_metrics
     if not value and len(new_possibles_choices) == 1:
         value = list(new_possibles_choices)[0]
 
-    return gr.update(choices=sorted(list(new_possibles_choices)), value=value)
+    return gr.Dropdown(choices=sorted(list(new_possibles_choices)), value=value)
 
 def reverse_search(base_folder: str, possible_datasets: List[str], grouping: str, metric_name: str) -> str:
     with ThreadPoolExecutor() as executor:
@@ -91,7 +88,7 @@ def reverse_search(base_folder: str, possible_datasets: List[str], grouping: str
 
 def reverse_search_add(datasets: List[str], reverse_search_results: str) -> List[str]:
     datasets = datasets or []
-    return sorted(list(set(datasets + reverse_search_results.strip().split("\n"))))
+    return list(set(datasets + reverse_search_results.strip().split("\n")))
 
 def metric_exists(base_folder: str, path: str, metric_name: str, group_by: str) -> bool:
     base_folder = get_datafolder(base_folder)
@@ -105,4 +102,39 @@ def load_metrics(base_folder: str, path: str, metric_name: str, group_by: str) -
         return MetricStatsDict.from_dict(json_metric)
 
 def load_data(dataset_path: str, base_folder: str, grouping: str, metric_name: str) -> MetricStatsDict:
-    return load_metrics(base_folder, dataset_path, metric_name, grouping)
+    return load_metrics(base_folder, dataset_path, metric_name, grouping)
+
+
+def fetch_graph_data(
+        base_folder: str,
+        datasets: List[str],
+        metric_name: str,
+        grouping: Grouping,
+        progress=gr.Progress(),
+):
+    if len(datasets) <= 0 or not metric_name or not grouping:
+        return None
+
+    with ThreadPoolExecutor() as pool:
+        data = list(
+            progress.tqdm(
+                pool.map(
+                    partial(load_data, base_folder=base_folder, metric_name=metric_name, grouping=grouping),
+                    datasets,
+                ),
+                total=len(datasets),
+                desc="Loading data...",
+            )
+        )
+
+    data = {path: result for path, result in zip(datasets, data)}
+    return data, None
+
+def update_datasets_with_regex(regex: str, selected_runs: List[str], all_runs: List[str]):
+    if not regex:
+        return []
+    new_dsts = {run for run in all_runs if re.search(regex, run)}
+    if not new_dsts:
+        return selected_runs
+    dst_union = new_dsts.union(selected_runs or [])
+    return sorted(list(dst_union))

src/logic/data_processing.py

@@ -1,3 +1,4 @@
+from datetime import datetime
 import json
 import re
 import heapq
@@ -7,6 +8,8 @@ from typing import Dict, Tuple, List, Literal
 import gradio as gr
 from datatrove.utils.stats import MetricStatsDict
 
+from src.logic.graph_settings import Grouping
+
 PARTITION_OPTIONS = Literal["Top", "Bottom", "Most frequent (n_docs)"]
 
 def prepare_for_non_grouped_plotting(metric: Dict[str, MetricStatsDict], normalization: bool, rounding: int) -> Dict[float, float]:
@@ -35,13 +38,14 @@ def prepare_for_group_plotting(metric: Dict[str, MetricStatsDict], top_k: int, d
     stds = [metric[key].standard_deviation for key in keys]
     return keys, means, stds
 
-def export_data(exported_data: Dict[str, MetricStatsDict], metric_name: str):
+def export_data(exported_data: Dict[str, MetricStatsDict], metric_name: str, grouping: Grouping):
     if not exported_data:
         return None
-    with tempfile.NamedTemporaryFile(mode="w", delete=False, prefix=metric_name, suffix=".json") as temp:
+
+    file_name = f"{metric_name}_{grouping}_{datetime.now().strftime('%Y-%m-%d_%H-%M-%S')}.json"
+    with open(file_name, 'w') as f:
         json.dump({
             name: sorted([{"value": key, **value} for key, value in dt.to_dict().items()], key=lambda x: x["value"])
             for name, dt in exported_data.items()
-        }, temp, indent=2)
-        temp_path = temp.name
-    return gr.update(visible=True, value=temp_path)
+        }, f, indent=2)
+    return gr.File(value=file_name, visible=True)

src/logic/graph_settings.py

@@ -0,0 +1,36 @@
+from typing import Literal
+import gradio as gr
+
+
+Grouping = Literal["histogram", "fqdn", "suffix", "summary"]
+
+def update_graph_options(grouping: Grouping):
+    """
+    Updates visibility of the graph options based on the grouping type.
+    The return should be in following order:
+    group_settings, histogram_settings
+    """
+    if grouping == "histogram":
+        return [
+            gr.TabItem(visible=False),
+            gr.TabItem(visible=True),
+            gr.TabItem(visible=False),
+        ]
+    elif grouping in ["fqdn", "suffix"]:
+        return [
+            gr.Column(visible=True),
+            gr.Column(visible=False),
+            gr.Column(visible=False),
+        ]
+    elif grouping == "summary":
+        return [
+            gr.Column(visible=False),
+            gr.Column(visible=False),
+            gr.Column(visible=True),
+    ]
+    
+    return [
+        gr.Column(visible=False),
+        gr.Column(visible=False),
+        gr.Column(visible=False),
+    ]

src/logic/plotting.py

@@ -4,8 +4,11 @@ import plotly.graph_objects as go
 import numpy as np
 import gradio as gr
 from typing import Dict, List
-from .data_processing import prepare_for_non_grouped_plotting, prepare_for_group_plotting
-from .utils import set_alpha
+
+from src.logic.data_processing import PARTITION_OPTIONS, prepare_for_non_grouped_plotting, prepare_for_group_plotting
+from src.logic.graph_settings import Grouping
+from src.logic.utils import set_alpha
+from datatrove.utils.stats import MetricStatsDict
 
 def plot_scatter(
         data: Dict[str, Dict[float, float]],
@@ -55,14 +58,15 @@ def plot_scatter(
     return fig
 
 def plot_bars(
-        data: Dict[str, List[Dict[str, float]]],
+        data: Dict[str, MetricStatsDict],
         metric_name: str,
         top_k: int,
-        direction: str,
+        direction: PARTITION_OPTIONS,
         regex: str | None,
         rounding: int,
         log_scale_x: bool,
         log_scale_y: bool,
+        show_stds: bool,
         progress: gr.Progress,
 ):
     fig = go.Figure()
@@ -77,7 +81,7 @@ def plot_bars(
             y=y,
             name=f"{name} Mean",
             marker=dict(color=set_alpha(px.colors.qualitative.Plotly[i % len(px.colors.qualitative.Plotly)], 0.5)),
-            error_y=dict(type='data', array=stds, visible=True)
+            error_y=dict(type='data', array=stds, visible=show_stds)
         ))
 
     fig.update_layout(
@@ -94,14 +98,65 @@ def plot_bars(
 
     return fig
 
-def plot_data(data, metric_name, normalization, rounding, grouping, top_k, direction, regex, log_scale_x, log_scale_y,
-              cumsum, perc, progress=gr.Progress()):
-    if rounding is None or top_k is None:
-        return None
-    graph_fc = (
-        partial(plot_scatter, normalization=normalization, rounding=rounding, cumsum=cumsum, perc=perc)
-        if grouping == "histogram"
-        else partial(plot_bars, top_k=top_k, direction=direction, regex=regex, rounding=rounding)
-    )
-    return graph_fc(data=data, metric_name=metric_name, progress=progress, log_scale_x=log_scale_x,
-                    log_scale_y=log_scale_y)
+
+# Add any other necessary functions
+
+def plot_data(
+    metric_data: Dict[str, MetricStatsDict],
+    metric_name: str,
+    normalize: bool,
+    rounding: int,
+    grouping: Grouping,
+    top_n: int,
+    direction: PARTITION_OPTIONS,
+    group_regex: str,
+    log_scale_x: bool,
+    log_scale_y: bool,
+    cdf: bool,
+    perc: bool,
+    show_stds: bool,
+) -> tuple[go.Figure, gr.Row, str]:
+    if grouping == "histogram":
+        fig = plot_scatter(
+            metric_data,
+            metric_name,
+            log_scale_x,
+            log_scale_y,
+            normalize,
+            rounding,
+            cdf,
+            perc,
+            gr.Progress(),
+        )
+        min_max_hist_data = generate_min_max_hist_data(metric_data)
+        return fig, gr.Row.update(visible=True), min_max_hist_data
+    else:
+        fig = plot_bars(
+            metric_data,
+            metric_name,
+            top_n,
+            direction,
+            group_regex,
+            rounding,
+            log_scale_x,
+            log_scale_y,
+            show_stds,
+            gr.Progress(),
+        )
+        return fig, gr.Row.update(visible=True), ""
+
+def generate_min_max_hist_data(data: Dict[str, MetricStatsDict]) -> str:
+    runs_data = {
+        run: {
+            "min": min(map(float, dato.keys())),
+            "max": max(map(float, dato.keys())),
+        }
+        for run, dato in data.items()
+    }
+    
+    runs_rows = [
+        f"| {run} | {values['min']:.4f} | {values['max']:.4f} |"
+        for run, values in runs_data.items()
+    ]
+    header = "| Run | Min | Max |\n|-----|-----|-----|\n"
+    return header + "\n".join(runs_rows)

src/view/help_tab.py

@@ -0,0 +1,37 @@
+import gradio as gr
+
+def create_help_tab():
+    gr.Markdown(
+        label="Readme",
+        value="""
+
+# Dataset Metrics Explorer
+## Features:
+- View metrics for various datasets you computed using datatrove
+- Search for metrics across datasets
+
+## View metrics Usage:
+1) Specify Metrics location (Stats block `output_folder`) and click "Fetch Datasets"
+2) Select datasets you are interested in using the dropdown or regex filter
+3) Specify Grouping (histogram/summary/fqdn/suffix) and Metric name
+4) Click "Render Metric", adjust Graph settings and see the result
+
+### Groupings:
+- **histogram**: Creates a line plot of values with their frequencies.
+    * normalize: Normalize the histogram to sum to 1
+    * CDF: Show the plot as cumulative distribution function
+    * %: Show the plot as percentage of the total
+- **(fqdn/suffix)**: Creates a bar plot of the avg. values of the metric for full qualifed domain name/suffix of domain.
+    * k: the number of groups to show
+    * Top/Bottom/Most frequent (n_docs): Groups with the top/bottom k values/most prevalant docs are shown
+- **summary**: Shows the average value of given metric for every dataset
+    * show_stds: Show the standard deviation from mean for every datasets
+
+## Reverse search Usage:
+To search for datasets containing a grouping and certain metric, use the Reverse search section.
+Specify the search parameters and click "Search". This will show you found datasets in the "Found datasets" textbox. You can modify the selection after search by removing unwanted lines and clicking "Add to selection".
+
+## Note:
+The data might not be 100% representative, due to the sampling and optimistic merging of the metrics (fqdn/suffix).
+"""
+        )

src/view/metric_view_tab.py

@@ -0,0 +1,222 @@
+from datetime import datetime
+import tempfile
+from typing import Callable
+import gradio as gr
+from functools import partial
+import re
+import json
+
+from src.logic.data_fetching import fetch_datasets, fetch_graph_data, fetch_groups, fetch_metrics, update_datasets_with_regex
+from src.logic.data_processing import export_data
+from src.logic.graph_settings import update_graph_options
+from src.logic.plotting import plot_data
+
+def create_metric_view_tab(METRICS_LOCATION_DEFAULT: str, available_datasets: gr.State, selected_datasets: gr.State):
+    metric_data = gr.State([])
+
+    with gr.Row():
+        with gr.Column(scale=2):
+            with gr.Row():
+                with gr.Column(scale=1):
+                    base_folder = gr.Textbox(
+                        label="Metrics Location",
+                        value=METRICS_LOCATION_DEFAULT,
+                    )
+                    datasets_fetch = gr.Button("Fetch Datasets")
+
+                with gr.Column(scale=1):
+                    regex_select = gr.Text(label="Regex filter", value=".*")
+                    regex_button = gr.Button("Search")
+            with gr.Row():
+                selected_datasets_dropdown = gr.Dropdown(
+                    choices=[],
+                    label="Datasets",
+                    multiselect=True,
+                    interactive=True,
+                )
+
+        with gr.Column(scale=1):
+            grouping_dropdown = gr.Dropdown(
+                choices=[],
+                label="Grouping",
+                multiselect=False,
+            )
+            metric_name_dropdown = gr.Dropdown(
+                choices=[],
+                label="Metric name",
+                multiselect=False,
+            )
+
+            render_button = gr.Button("Render Metric", variant="primary")
+
+    with gr.Tabs():
+        with gr.TabItem("Graph Settings"):
+            log_scale_x_checkbox = gr.Checkbox(
+                label="Log scale x",
+                value=False,
+            )
+            log_scale_y_checkbox = gr.Checkbox(
+                label="Log scale y",
+                value=False,
+            )
+            rounding = gr.Number(
+                label="Rounding",
+                value=2,
+            )
+
+        with gr.TabItem("Grouping Settings") as group_settings:
+            with gr.Row() as group_choices:
+                with gr.Column(scale=2):
+                    group_regex = gr.Text(
+                        label="Group Regex",
+                        value=None,
+                    )
+                    with gr.Row():
+                        top_select = gr.Number(
+                            label="N Groups",
+                            value=100,
+                            interactive=True,
+                        )
+
+                        direction_checkbox = gr.Radio(
+                            label="Partition",
+                            choices=[
+                                "Top",
+                                "Bottom",
+                                "Most frequent (n_docs)",
+                            ],
+                            value="Most frequent (n_docs)",
+                        )
+
+        with gr.TabItem("Histogram Settings") as histogram_settings:
+            normalization_checkbox = gr.Checkbox(
+                label="Normalize",
+                value=True,
+                visible=False
+            )
+            cdf_checkbox = gr.Checkbox(
+                label="CDF",
+                value=False,
+            )
+            perc_checkbox = gr.Checkbox(
+                label="%",
+                value=False,
+            )
+
+        with gr.TabItem("Summary Settings") as summary_settings:
+            show_stds_checkbox = gr.Checkbox(
+                label="Show standard deviations",
+                value=False,
+            )
+
+    with gr.Row():
+        graph_output = gr.Plot(label="Graph")
+    with gr.Row(visible=False) as min_max_hist:
+        with gr.Column(scale=3):
+            min_max_hist_data = gr.Markdown()
+        with gr.Column(scale=1):
+            export_data_button = gr.Button("Export Data")
+            export_data_json = gr.File(visible=False)
+            
+
+
+    def update_selected_datasets_dropdown(available_datasets, selected_datasets):
+        return gr.Dropdown(choices=available_datasets, value=sorted(selected_datasets))
+
+
+    datasets_fetch.click(
+        fn=fetch_datasets,
+        inputs=[base_folder],
+        outputs=[available_datasets],
+    )
+
+    available_datasets.change(
+        fn=update_selected_datasets_dropdown,
+        inputs=[available_datasets, selected_datasets],
+        outputs=selected_datasets_dropdown,
+    )
+
+    regex_button.click(
+        fn=update_datasets_with_regex,
+        inputs=[regex_select, selected_datasets, available_datasets],
+        outputs=selected_datasets,
+    )
+    
+    def update_selected_datasets(selected_datasets_dropdown):
+        return selected_datasets_dropdown
+    
+    selected_datasets_dropdown.change(
+        fn=update_selected_datasets,
+        inputs=[selected_datasets_dropdown],
+        outputs=selected_datasets,
+    )
+    
+    selected_datasets.change(
+        fn=update_selected_datasets_dropdown,
+        inputs=[available_datasets, selected_datasets],
+        outputs=selected_datasets_dropdown,
+    )
+
+
+    selected_datasets.change(
+        fn=fetch_groups,
+        inputs=[base_folder, selected_datasets, grouping_dropdown],
+        outputs=grouping_dropdown,
+    )
+
+    grouping_dropdown.change(
+        fn=fetch_metrics,
+        inputs=[base_folder, selected_datasets, grouping_dropdown, metric_name_dropdown],
+        outputs=metric_name_dropdown,
+    )
+
+    render_button.click(
+        fn=fetch_graph_data,
+        inputs=[
+            base_folder,
+            selected_datasets,
+            metric_name_dropdown,
+            grouping_dropdown,
+        ],
+        # We also output the graph_output = None to show the progress
+        outputs=[metric_data, graph_output],
+    )
+
+
+    grouping_dropdown.change(
+        fn=update_graph_options,
+        inputs=[grouping_dropdown],
+        outputs=[group_settings, histogram_settings, summary_settings],
+    )
+
+
+    gr.on(
+        triggers=[normalization_checkbox.input, rounding.input, group_regex.input, direction_checkbox.input,
+                  top_select.input, log_scale_x_checkbox.input,
+                  log_scale_y_checkbox.input, cdf_checkbox.input, perc_checkbox.input, show_stds_checkbox.input, metric_data.change],
+        fn=plot_data,
+        inputs=[
+            metric_data,
+            metric_name_dropdown,
+            normalization_checkbox,
+            rounding,
+            grouping_dropdown,
+            top_select,
+            direction_checkbox,
+            group_regex,
+            log_scale_x_checkbox,
+            log_scale_y_checkbox,
+            cdf_checkbox,
+            perc_checkbox,
+            show_stds_checkbox
+        ],
+        outputs=[graph_output, min_max_hist, min_max_hist_data],
+    )
+
+    export_data_button.click(
+        fn=export_data,
+        inputs=[metric_data, metric_name_dropdown, grouping_dropdown],
+        outputs=[export_data_json],
+    )
+    
+    return base_folder

src/view/reverse_search_tab.py

@@ -0,0 +1,53 @@
+from functools import partial
+import gradio as gr
+
+from src.logic.data_fetching import fetch_groups, fetch_metrics, reverse_search, reverse_search_add
+
+def create_reverse_search_tab(base_folder: gr.Textbox, datasets_available: gr.State, datasets_selected: gr.State):
+    reverse_search_headline = gr.Markdown(value="# Reverse Metrics Search")
+
+    with gr.Row():
+        with gr.Column(scale=1):
+            reverse_grouping_dropdown = gr.Dropdown(
+                choices=[],
+                label="Grouping",
+                multiselect=False,
+            )
+            reverse_metric_name_dropdown = gr.Dropdown(
+                choices=[],
+                label="Metric Name",
+                multiselect=False,
+            )
+            reverse_search_button = gr.Button("Search")
+            reverse_search_add_button = gr.Button("Add to selection")
+
+        with gr.Column(scale=2):
+            reverse_search_results = gr.Textbox(
+                label="Found datasets",
+                lines=10,
+                placeholder="Found datasets containing the group/metric name. You can modify the selection after search by removing unwanted lines and clicking Add to selection"
+            )
+            
+    datasets_available.change(
+        fn=partial(fetch_groups, type="union"),
+        inputs=[base_folder, datasets_available, reverse_grouping_dropdown],
+        outputs=[reverse_grouping_dropdown],
+    )
+
+    reverse_grouping_dropdown.select(
+        fn=partial(fetch_metrics, type="union"),
+        inputs=[base_folder, datasets_available, reverse_grouping_dropdown, reverse_metric_name_dropdown],
+        outputs=reverse_metric_name_dropdown,
+    )
+
+    reverse_search_button.click(
+        fn=partial(reverse_search),
+        inputs=[base_folder, datasets_available, reverse_grouping_dropdown, reverse_metric_name_dropdown],
+        outputs=reverse_search_results,
+    )
+
+    reverse_search_add_button.click(
+        fn=reverse_search_add,
+        inputs=[datasets_selected, reverse_search_results],
+        outputs=datasets_selected,
+    )

src/view/view.py

@@ -1,325 +1,28 @@
+import os
 import gradio as gr
-from src.logic.data_fetching import fetch_datasets, fetch_groups, fetch_metrics, load_data, reverse_search, reverse_search_add
-from src.logic.data_processing import export_data
-from src.logic.plotting import plot_data
-from src.logic.utils import get_desc
-from concurrent.futures import ThreadPoolExecutor
 from functools import partial
-import os
 import re
-
-
+from src.view.help_tab import create_help_tab
+from src.view.metric_view_tab import create_metric_view_tab
+from src.view.reverse_search_tab import create_reverse_search_tab
+from src.logic.data_fetching import fetch_datasets, fetch_groups, fetch_metrics, reverse_search, reverse_search_add
 
 METRICS_LOCATION_DEFAULT = os.getenv("METRICS_LOCATION_DEFAULT", "hf://datasets/HuggingFaceFW-Dev/summary-stats-files")
 
-def update_graph(
-        base_folder,
-        datasets,
-        metric_name,
-        grouping,
-        log_scale_x,
-        log_scale_y,
-        rounding,
-        normalization,
-        top_k,
-        direction,
-        regex,
-        cumsum,
-        perc,
-        progress=gr.Progress(),
-):
-    if len(datasets) <= 0 or not metric_name or not grouping:
-        return None
-
-    with ThreadPoolExecutor() as pool:
-        data = list(
-            progress.tqdm(
-                pool.map(
-                    partial(load_data, base_folder=base_folder, metric_name=metric_name, grouping=grouping),
-                    datasets,
-                ),
-                total=len(datasets),
-                desc="Loading data...",
-            )
-        )
-
-    data = {path: result for path, result in zip(datasets, data)}
-    return plot_data(data, metric_name, normalization, rounding, grouping, top_k, direction, regex, log_scale_x,
-                     log_scale_y, cumsum, perc, progress), data, export_data(data, metric_name), get_desc(data)
-
 def create_interface():
     with gr.Blocks() as demo:
-        datasets = gr.State([])
-        exported_data = gr.State([])
         metrics_headline = gr.Markdown(value="# Metrics Exploration")
+        available_datasets = gr.State([])
+        selected_datasets = gr.State([])
 
         with gr.Tabs():
-            with gr.TabItem("Help"):
-                gr.Markdown(
-                    label="Readme",
-                    value="""
-## How to use:
-1) Specify Metrics location (Stats block `output_folder` without the last path segment) and click "Fetch Datasets"
-2) Select datasets you are interested in using the dropdown or regex filter
-3) Specify Grouping (global average/value/fqdn/suffix) and Metric name
-4) Click "Render Metric" 
-
-
-## Groupings:
-- **histogram**: Creates a line plot of values with their frequencies. If normalization is on, the frequencies sum to 1.
-    * normalize:
-- **(fqdn/suffix)**: Creates a bar plot of the avg. values of the metric for full qualifed domain name/suffix of domain.
-    * k: the number of groups to show
-    * Top/Bottom/Most frequent (n_docs): Groups with the top/bottom k values/most prevalant docs are shown
-- **none**: Shows the average value of given metric
-
-## Reverse search:
-To search for datasets containing a grouping and certain metric, use the Reverse search section.
-Specify the search parameters and click "Search". This will show you found datasets in the "Found datasets" textbox. You can modify the selection after search by removing unwanted lines and clicking "Add to selection".
-
-## Note:
-The data might not be 100% representative, due to the sampling and optimistic merging of the metrics (fqdn/suffix).
-""",
-                )
-
+            with gr.Tab("Help"):
+                create_help_tab()
+            
             with gr.TabItem("Metric View"):
-                with gr.Row():
-                    with gr.Column(scale=2):
-                        with gr.Row():
-                            with gr.Column(scale=1):
-                                base_folder = gr.Textbox(
-                                    label="Metrics Location",
-                                    value=METRICS_LOCATION_DEFAULT,
-                                )
-                                datasets_refetch = gr.Button("Fetch Datasets")
-
-                            with gr.Column(scale=1):
-                                regex_select = gr.Text(label="Regex filter", value=".*")
-                                regex_button = gr.Button("Search")
-                        with gr.Row():
-                            datasets_selected = gr.Dropdown(
-                                choices=[],
-                                label="Datasets",
-                                multiselect=True,
-                            )
-
-                    with gr.Column(scale=1):
-                        grouping_dropdown = gr.Dropdown(
-                            choices=[],
-                            label="Grouping",
-                            multiselect=False,
-                        )
-                        metric_name_dropdown = gr.Dropdown(
-                            choices=[],
-                            label="Metric name",
-                            multiselect=False,
-                        )
-
-                        render_button = gr.Button("Render Metric", variant="primary")
-
-                with gr.Tabs():
-                    with gr.TabItem("Graph Settings"):
-                        log_scale_x_checkbox = gr.Checkbox(
-                            label="Log scale x",
-                            value=False,
-                        )
-                        log_scale_y_checkbox = gr.Checkbox(
-                            label="Log scale y",
-                            value=False,
-                        )
-                        rounding = gr.Number(
-                            label="Rounding",
-                            value=2,
-                        )
-                        normalization_checkbox = gr.Checkbox(
-                            label="Normalize",
-                            value=True,
-                            visible=False
-                        )
-                        with gr.Row():
-                            export_data_json = gr.File(visible=False)
-
-                    with gr.TabItem("Grouping Settings"):
-                        with gr.Row(visible=False) as group_choices:
-                            with gr.Column(scale=2):
-                                group_regex = gr.Text(
-                                    label="Group Regex",
-                                    value=None,
-                                )
-                                with gr.Row():
-                                    top_select = gr.Number(
-                                        label="N Groups",
-                                        value=100,
-                                        interactive=True,
-                                    )
-
-                                    direction_checkbox = gr.Radio(
-                                        label="Partition",
-                                        choices=[
-                                            "Top",
-                                            "Bottom",
-                                            "Most frequent (n_docs)",
-                                        ],
-                                        value="Most frequent (n_docs)",
-                                    )
-
-                    with gr.TabItem("Histogram Settings") as histogram_settings:
-                        cdf_checkbox = gr.Checkbox(
-                            label="CDF",
-                            value=False,
-                        )
-                        perc_checkbox = gr.Checkbox(
-                            label="%",
-                            value=False,
-                        )
-                        with gr.Column(visible=False) as min_max_hist:
-                            min_max_hist_data = gr.Markdown()
-
-                with gr.Row():
-                    graph_output = gr.Plot(label="Graph")
+                base_folder = create_metric_view_tab(METRICS_LOCATION_DEFAULT, available_datasets, selected_datasets)
 
             with gr.TabItem("Reverse Metrics Search"):
-                reverse_search_headline = gr.Markdown(value="# Reverse Metrics Search")
-
-                with gr.Row():
-                    with gr.Column(scale=1):
-                        reverse_grouping_dropdown = gr.Dropdown(
-                            choices=[],
-                            label="Grouping",
-                            multiselect=False,
-                        )
-                        reverse_metric_name_dropdown = gr.Dropdown(
-                            choices=[],
-                            label="Metric Name",
-                            multiselect=False,
-                        )
-                        reverse_search_button = gr.Button("Search")
-                        reverse_search_add_button = gr.Button("Add to selection")
-
-                    with gr.Column(scale=2):
-                        reverse_search_results = gr.Textbox(
-                            label="Found datasets",
-                            lines=10,
-                            placeholder="Found datasets containing the group/metric name. You can modify the selection after search by removing unwanted lines and clicking Add to selection"
-                        )
-
-        render_button.click(
-            fn=update_graph,
-            inputs=[
-                base_folder,
-                datasets_selected,
-                metric_name_dropdown,
-                grouping_dropdown,
-                log_scale_x_checkbox,
-                log_scale_y_checkbox,
-                rounding,
-                normalization_checkbox,
-                top_select,
-                direction_checkbox,
-                group_regex,
-                cdf_checkbox,
-                perc_checkbox
-            ],
-            outputs=[graph_output, exported_data, export_data_json, min_max_hist_data],
-        )
-
-        gr.on(
-            triggers=[normalization_checkbox.change, rounding.change, group_regex.change, direction_checkbox.change,
-                      top_select.change, log_scale_x_checkbox.change,
-                      log_scale_y_checkbox.change, cdf_checkbox.change, perc_checkbox.change],
-            fn=plot_data,
-            inputs=[
-                exported_data,
-                metric_name_dropdown,
-                normalization_checkbox,
-                rounding,
-                grouping_dropdown,
-                top_select,
-                direction_checkbox,
-                group_regex,
-                log_scale_x_checkbox,
-                log_scale_y_checkbox,
-                cdf_checkbox,
-                perc_checkbox
-            ],
-            outputs=[graph_output],
-        )
-
-        datasets_selected.change(
-            fn=fetch_groups,
-            inputs=[base_folder, datasets_selected, grouping_dropdown],
-            outputs=grouping_dropdown,
-        )
-
-        grouping_dropdown.change(
-            fn=fetch_metrics,
-            inputs=[base_folder, datasets_selected, grouping_dropdown, metric_name_dropdown],
-            outputs=metric_name_dropdown,
-        )
-
-        reverse_grouping_dropdown.select(
-            fn=partial(fetch_metrics, type="union"),
-            inputs=[base_folder, datasets, reverse_grouping_dropdown, reverse_metric_name_dropdown],
-            outputs=reverse_metric_name_dropdown,
-        )
-
-        reverse_search_button.click(
-            fn=reverse_search,
-            inputs=[base_folder, datasets, reverse_grouping_dropdown, reverse_metric_name_dropdown],
-            outputs=reverse_search_results,
-        )
-
-        reverse_search_add_button.click(
-            fn=reverse_search_add,
-            inputs=[datasets_selected, reverse_search_results],
-            outputs=datasets_selected,
-        )
-
-        datasets_refetch.click(
-            fn=fetch_datasets,
-            inputs=[base_folder],
-            outputs=[datasets, datasets_selected, reverse_grouping_dropdown],
-        )
-
-
-        def update_datasets_with_regex(regex, selected_runs, all_runs):
-            if not regex:
-                return
-            new_dsts = {run for run in all_runs if re.search(regex, run)}
-            if not new_dsts:
-                return gr.update(value=list(selected_runs))
-            dst_union = new_dsts.union(selected_runs or [])
-            return gr.update(value=sorted(list(dst_union)))
-
-
-        regex_button.click(
-            fn=update_datasets_with_regex,
-            inputs=[regex_select, datasets_selected, datasets],
-            outputs=datasets_selected,
-        )
-
-
-        def update_grouping_options(grouping):
-            if grouping == "histogram":
-                return {
-                    normalization_checkbox: gr.Column(visible=True),
-                    group_choices: gr.Column(visible=False),
-                    min_max_hist: gr.Column(visible=True),
-                    histogram_settings: gr.TabItem(visible=True),
-                }
-            else:
-                return {
-                    normalization_checkbox: gr.Column(visible=False),
-                    group_choices: gr.Column(visible=True),
-                    min_max_hist: gr.Column(visible=False),
-                    histogram_settings: gr.TabItem(visible=False),
-                }
-
-
-        grouping_dropdown.change(
-            fn=update_grouping_options,
-            inputs=[grouping_dropdown],
-            outputs=[normalization_checkbox, group_choices, min_max_hist, histogram_settings],
-        )
+                create_reverse_search_tab(base_folder, available_datasets, selected_datasets)
 
     return demo