Initital commit

README.md

@@ -1,13 +1,18 @@
 ---
-title: Data Contamination Report
-emoji: 🏃
+title: 🐍💨 Data Contamination Report
+emoji: 🏭
 colorFrom: green
 colorTo: blue
 sdk: gradio
-sdk_version: 4.21.0
+python_version: 3.10
+sdk_version: 4.19.1
 app_file: app.py
+app_port: 7860
+fullWidth: true
 pinned: false
-license: apache-2.0
+license: mit
+suggested_hardware: cpu-upgrade
+
 ---
 
 Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,181 @@
+import gradio as gr
+import pandas as pd
+
+from dataset import get_dataframe
+from markdown import GUIDELINES, PANEL_MARKDOWN
+
+
+df = get_dataframe()
+
+
+def filter_dataframe(dataframe, eval_dataset, cont_source, checkboxes):
+    """
+    Filter the dataframe based on the provided evaluation dataset, contaminated source, and checkboxes.
+
+    Args:
+        dataframe (pandas.DataFrame): The input dataframe to filter.
+        eval_dataset (str): The evaluation dataset to filter by.
+        cont_source (str): The contaminated source to filter by.
+        checkboxes (list): The checkboxes to filter by.
+
+    Returns:
+        pandas.DataFrame: The filtered dataframe.
+    """
+    if isinstance(eval_dataset, str):
+        dataframe = dataframe[
+            dataframe["Evaluation Dataset"].str.contains(eval_dataset)
+        ]
+    if isinstance(cont_source, str):
+        dataframe = dataframe[
+            dataframe["Contaminated Source"].str.contains(cont_source)
+        ]
+    if isinstance(checkboxes, list) and "Exclude model-based evidences" in checkboxes:
+        dataframe = dataframe[dataframe["Approach"] != "model-based"]
+    if isinstance(checkboxes, list) and "Show only contaminated" in checkboxes:
+        dataframe = dataframe[
+            (dataframe["Train Split"] > 0.0)
+            | (dataframe["Development Split"] > 0.0)
+            | (dataframe["Test Split"] > 0.0)
+        ]
+
+    return dataframe
+
+
+def filter_dataframe_corpus(*args, **kwargs) -> pd.DataFrame:
+    """
+    Filter the dataframe for corpus contamination.
+
+    Returns:
+        pandas.DataFrame: The filtered dataframe for corpus contamination.
+    """
+    # Get rows in which the column Model or corpus is equal to dataset
+    filtered_df = df[df["Model or corpus"] == "corpus"]
+    filtered_df = filtered_df.drop(columns=["Model or corpus"])
+    return filter_dataframe(filtered_df, *args, **kwargs)
+
+
+def filter_dataframe_model(*args, **kwargs) -> pd.DataFrame:
+    """
+    Filter the dataframe for model contamination.
+
+    Returns:
+        pandas.DataFrame: The filtered dataframe for model contamination.
+    """
+    # Get rows in which the column Model or corpus is equal to dataset
+    filtered_df = df[df["Model or corpus"] == "model"]
+    filtered_df = filtered_df.drop(columns=["Model or corpus"])
+    return filter_dataframe(filtered_df, *args, **kwargs)
+
+
+theme = gr.themes.Soft(
+    primary_hue="emerald",
+    secondary_hue="red",
+    text_size="sm",
+    spacing_size="sm",
+    font=[
+        gr.themes.GoogleFont("Poppins"),
+        gr.themes.GoogleFont("Poppins"),
+        gr.themes.GoogleFont("Poppins"),
+        gr.themes.GoogleFont("Poppins"),
+    ],
+).set(block_background_fill="*neutral_50", block_background_fill_dark="*neutral_950")
+
+
+with gr.Blocks(
+    theme=theme,
+    title="💨 Data Contamination Report",
+    analytics_enabled=False,
+) as demo:
+    gr.Markdown(PANEL_MARKDOWN)
+    with gr.Tab("Corpus contamination") as tab_corpus:
+        with gr.Row(variant="compact"):
+            with gr.Column():
+                eval_dataset_corpus = gr.Textbox(
+                    placeholder="Evaluation dataset",
+                    label="Evaluation dataset",
+                    value="",
+                )
+                cont_corpora = gr.Textbox(
+                    placeholder="Pre-training corpora",
+                    label="Pre-training corpora",
+                    value="",
+                )
+            with gr.Column():
+                checkboxes_corpus = gr.CheckboxGroup(
+                    ["Exclude model-based evidences", "Show only contaminated"],
+                    label="Search options",
+                    value=[],
+                )
+
+        filter_corpus_btn = gr.Button("Filter")
+
+        corpus_dataframe = gr.DataFrame(
+            value=filter_dataframe_corpus(
+                eval_dataset_corpus, cont_corpora, checkboxes_corpus
+            ).style.format(precision=2),
+            headers=df.columns.to_list(),
+            datatype=[
+                "markdown",
+                "markdown",
+                "number",
+                "number",
+                "number",
+                "str",
+                "markdown",
+                "markdown",
+            ],
+        )
+
+    with gr.Tab("Model contamination") as tab_model:
+        with gr.Row(variant="compact"):
+            with gr.Column():
+                eval_dataset_model = gr.Textbox(
+                    placeholder="Evaluation dataset",
+                    label="Evaluation dataset",
+                    value="",
+                )
+                cont_model = gr.Textbox(
+                    placeholder="Model", label="Pre-training corpora", value=""
+                )
+            with gr.Column():
+                checkboxes_model = gr.CheckboxGroup(
+                    ["Exclude model-based evidences", "Show only contaminated"],
+                    label="Search options",
+                    value=[],
+                )
+
+        filter_model_btn = gr.Button("Filter")
+
+        model_dataframe = gr.DataFrame(
+            value=filter_dataframe_model(
+                eval_dataset_model, cont_model, checkboxes_model
+            ),
+            headers=df.columns.to_list(),
+            datatype=[
+                "markdown",
+                "markdown",
+                "number",
+                "number",
+                "number",
+                "str",
+                "markdown",
+                "markdown",
+            ],
+        )
+
+    filter_corpus_btn.click(
+        filter_dataframe_corpus,
+        inputs=[eval_dataset_corpus, cont_corpora, checkboxes_corpus],
+        outputs=corpus_dataframe,
+    )
+    filter_model_btn.click(
+        filter_dataframe_model,
+        inputs=[eval_dataset_model, cont_model, checkboxes_model],
+        outputs=model_dataframe,
+    )
+
+    with gr.Tab("Guidelines") as tab_guidelines:
+        gr.Markdown(GUIDELINES)
+
+
+demo.launch()

contamination_report.csv

@@ -0,0 +1,4 @@
+Evaluation Dataset;Contaminated Source;Model or corpus;Train Split;Development Split;Test Split;Approach;Citation;PR Link
+conll2003;google/gemma-7b;model;1.0;1.0;1.0;model-based;https://hitz-zentroa.github.io/lm-contamination/blog/;
+conll2003;EleutherAI/the_pile_deduplicated;corpus;1.0;1.0;1.0;data-based;https://aclanthology.org/2023.findings-emnlp.722/;www.google.com
+Test;lololol;corpus;1.0;1.0;1.0;data-based;https://arxiv.org/abs/2310.03668;

dataset.py

@@ -0,0 +1,260 @@
+import json
+import os
+
+import filelock
+import huggingface_hub
+import pandas as pd
+
+from utils import (
+    build_datasets_urls,
+    build_models_urls,
+    build_text_icon,
+    download_favicons,
+    get_base_url,
+    get_domain_name,
+)
+
+
+HF_ICON = "https://huggingface.co/front/assets/huggingface_logo.svg"
+CROSS_ICON = "https://upload.wikimedia.org/wikipedia/commons/4/4e/Cross.png"
+
+DISABLE_ONLINE_CACHE = False
+ONLINE_CACHE = "CONDA-Workshop/RequestCache"
+
+
+def save_cache(cache_data, cache_file, initial_timestamp):
+    print(f"Saving cache to {cache_file}")
+    # Acquire lock before reading and updating the file to prevent race conditions
+    with filelock.FileLock(f"{cache_file}.lock"):
+        # Check if the file has been modified since the initial read
+        current_timestamp = (
+            os.path.getmtime(cache_file) if os.path.exists(cache_file) else None
+        )
+        if current_timestamp is None or initial_timestamp != current_timestamp:
+            # File has been modified or created since initial read, re-read the file
+            try:
+                with open(cache_file, "r", encoding="utf8") as f:
+                    # Update the dictionary with newly added entries
+                    cache_dict = json.load(f)
+                    # Test if cache_dict and cache_data are different
+                    if cache_dict != cache_data:
+                        cache_data.update(cache_dict)
+
+            except FileNotFoundError:
+                pass  # If the file doesn't exist at this point, continue with the current dictionary
+
+        # Write the updated dictionary back to the file
+        with open(cache_file, "w", encoding="utf8") as f:
+            json.dump(cache_data, f, ensure_ascii=False, indent=4)
+
+        if not DISABLE_ONLINE_CACHE:
+            try:
+                huggingface_hub.upload_file(
+                    repo_id=ONLINE_CACHE,
+                    repo_type="dataset",
+                    token=os.environ.get("TOKEN") or True,
+                    path_in_repo=cache_file,
+                    path_or_fileobj=cache_file,
+                )
+            except Exception as e:
+                print(f"Unable to upload {cache_file}: {e}")
+
+    return cache_data
+
+
+def update_favicon_cache(sources):
+    # Load the favicon dictionary if it exists
+    favicon_dict = {}
+    favicon_file_path = "favicons.json"
+    initial_timestamp = None
+
+    if not DISABLE_ONLINE_CACHE:
+        try:
+            huggingface_hub.hf_hub_download(
+                repo_id=ONLINE_CACHE,
+                repo_type="dataset",
+                token=os.environ.get("TOKEN") or True,
+                filename=favicon_file_path,
+                local_dir=os.getcwd(),
+            )
+        except Exception as e:
+            print(f"Unable to download favicons.json: {e}")
+
+    # Attempt to load the favicon dictionary and record its last modification time
+    if os.path.exists(favicon_file_path):
+        initial_timestamp = os.path.getmtime(favicon_file_path)
+        try:
+            with open(favicon_file_path, "r", encoding="utf8") as f:
+                favicon_dict = json.load(f)
+        except FileNotFoundError:
+            pass  # File not found, proceed with an empty dictionary
+
+    # Determine which favicons need to be downloaded
+    missing_domains = [domain for domain in sources if domain not in favicon_dict]
+
+    # Download missing favicons in batch
+    if missing_domains:
+        new_favicon_urls = download_favicons(missing_domains)
+        favicon_dict.update(new_favicon_urls)
+        favicon_dict = save_cache(
+            cache_data=favicon_dict,
+            cache_file=favicon_file_path,
+            initial_timestamp=initial_timestamp,
+        )
+
+    return favicon_dict
+
+
+def update_model_url_cache(models):
+    models = [x for x in models if x is not None]
+    models = list(set(models))
+
+    # Load the model url dictionary if it exists
+    model_url_dict = {}
+    model_url_file_path = "model_urls.json"
+    initial_timestamp = None
+
+    if not DISABLE_ONLINE_CACHE:
+        try:
+            huggingface_hub.hf_hub_download(
+                repo_id=ONLINE_CACHE,
+                repo_type="dataset",
+                token=os.environ.get("TOKEN") or True,
+                filename=model_url_file_path,
+                local_dir=os.getcwd(),
+            )
+        except Exception as e:
+            print(f"Unable to download model_urls.json: {e}")
+
+    # Attempt to load the model url dictionary and record its last modification time
+    if os.path.exists(model_url_file_path):
+        initial_timestamp = os.path.getmtime(model_url_file_path)
+        try:
+            with open(model_url_file_path, "r", encoding="utf8") as f:
+                model_url_dict = json.load(f)
+        except FileNotFoundError:
+            pass  # File not found, proceed with an empty dictionary
+
+    # Determine which model urls need to be downloaded
+    missing_model_urls = [model for model in models if model not in model_url_dict]
+
+    # Download missing model urls in batch
+    if missing_model_urls:
+        new_model_urls = build_models_urls(missing_model_urls)
+        model_url_dict.update(new_model_urls)
+        model_url_dict = save_cache(
+            cache_data=model_url_dict,
+            cache_file=model_url_file_path,
+            initial_timestamp=initial_timestamp,
+        )
+
+    return model_url_dict
+
+
+def update_dataset_url_cache(datasets):
+    datasets = [x for x in datasets if x is not None]
+    datasets = list(set(datasets))
+
+    # Load the dataset url dictionary if it exists
+    dataset_url_dict = {}
+    dataset_url_file_path = "dataset_urls.json"
+    initial_timestamp = None
+
+    if not DISABLE_ONLINE_CACHE:
+        try:
+            huggingface_hub.hf_hub_download(
+                repo_id=ONLINE_CACHE,
+                repo_type="dataset",
+                token=os.environ.get("TOKEN") or True,
+                filename=dataset_url_file_path,
+                local_dir=os.getcwd(),
+            )
+        except Exception as e:
+            print(f"Unable to download dataset_urls.json: {e}")
+
+    # Attempt to load the dataset url dictionary and record its last modification time
+    if os.path.exists(dataset_url_file_path):
+        initial_timestamp = os.path.getmtime(dataset_url_file_path)
+        try:
+            with open(dataset_url_file_path, "r", encoding="utf8") as f:
+                dataset_url_dict = json.load(f)
+        except FileNotFoundError:
+            pass  # File not found, proceed with an empty dictionary
+
+    # Determine which dataset urls need to be downloaded
+    missing_dataset_urls = [
+        dataset for dataset in datasets if dataset not in dataset_url_dict
+    ]
+
+    # Download missing dataset urls in batch
+    if missing_dataset_urls:
+        new_dataset_urls = build_datasets_urls(missing_dataset_urls)
+        dataset_url_dict.update(new_dataset_urls)
+        dataset_url_dict = save_cache(
+            cache_data=dataset_url_dict,
+            cache_file=dataset_url_file_path,
+            initial_timestamp=initial_timestamp,
+        )
+
+    return dataset_url_dict
+
+
+def get_dataframe():
+    # Load the contamination_report.csv file
+    data = pd.read_csv("contamination_report.csv", delimiter=";", header=0)
+
+    # Load the favicon dictionary if it exists
+    favicon_dict = {}
+
+    # Update the favicon dictionary
+    favicon_dict = update_favicon_cache([get_base_url(x) for x in data["Citation"]])
+
+    # Update the model url dictionary
+    model_url_dict = update_model_url_cache(
+        data[data["Model or corpus"] == "model"]["Contaminated Source"]
+    )
+
+    # Update the dataset url dictionary
+    dataset_url_dict = update_dataset_url_cache(
+        list(data["Evaluation Dataset"])
+        + list(data[data["Model or corpus"] == "corpus"]["Contaminated Source"])
+    )
+
+    # Add favicons URLs to the dataframe in a vectorized manner
+    data["Citation"] = data["Citation"].apply(
+        lambda x: build_text_icon(
+            text=get_domain_name(x),
+            url=x,
+            icon_url=favicon_dict.get(get_base_url(x), ""),
+        )
+    )
+
+    data["PR Link"] = data["PR Link"].apply(
+        lambda x: build_text_icon(
+            text="",
+            url=x if x == x else "no link",
+            icon_url=HF_ICON if x == x else CROSS_ICON,
+        )
+    )
+
+    data["Evaluation Dataset"] = data["Evaluation Dataset"].apply(
+        lambda x: build_text_icon(
+            text=x,
+            url=dataset_url_dict.get(x, ""),
+            icon_url=HF_ICON,
+        )
+    )
+
+    # For "Contaminated Source" use build_dataset_url if "Model or corpus" is "corpus" and build_model_url if "Model or corpus" is "model"
+    data["Contaminated Source"] = data.apply(
+        lambda x: build_text_icon(
+            text=x["Contaminated Source"],
+            url=dataset_url_dict.get(x["Contaminated Source"], "")
+            if x["Model or corpus"] == "corpus"
+            else model_url_dict.get(x["Contaminated Source"], ""),
+            icon_url=HF_ICON,
+        ),
+        axis=1,
+    )
+
+    return data

markdown.py

@@ -0,0 +1,69 @@
+GUIDELINES = """
+# Contribution Guidelines  
+  
+The 💨Data Contamination Report  is a community-driven project and we welcome contributions from everyone.The objetive of this project is to provide a comprehensive list of data contamination cases, for both models and datasets. We aim to provide a tool for the community for avoiding evaluating  
+models on contaminated datasets. We also expect to generate a dataset that will help researchers  
+to develop algorithms to automatically detect contaminated datasets in the future.  
+  
+If you wish to contribute to the project by reporting a data contamination case, please open a pull request  
+in the [✋Community Tab](https://huggingface.co/spaces/CONDA-Workshop/Data-Contamination-Report/discussions).Your pull request should edit the [contamination_report.csv](https://huggingface.co/spaces/CONDA-Workshop/Data-Contamination-Report/blob/main/contamination_report.csv)  
+file and add a new row with the details of the contamination case. Please will the following template with the details of the contamination case.  ***Pull Requests that do not follow the template won't be accepted.***
+
+# Template for reporting data contamination  
+  
+```markdown
+## What are you reporting:  
+- [ ] Evaluation dataset(s) found in a pre-training corpus. (e.g. COPA found in ThePile) 
+- [ ] Evaluation dataset(s) found in a pre-trained model. (e.g. FLAN T5 has been trained on ANLI)  
+
+**Evaluation dataset(s)**: Name(s) of the evaluation dataset(s). If available in the HuggingFace Hub please write the path  (e.g. `uonlp/CulturaX`), otherwise provide a link to a paper, GitHub or dataset-card.  
+  
+**Contaminated model(s)**: Name of the model(s) (if any) that have been contaminated with the evaluation dataset. If available in the HuggingFace Hub please list the corresponding paths (e.g. `allenai/OLMo-7B`).  
+  
+**Contaminated corpora**: Name of the corpora used to pretrain models (if any) that have been contaminated with the evaluation dataset. If available in the HuggingFace hub please write the path (e.g. `CohereForAI/aya_dataset`)   
+
+**Contaminated split(s)**: If the dataset has Train, Development and/or Test splits please report the contaminated split(s). You can report a percentage of the dataset contaminated. 
+
+  
+## Briefly describe your method to detect data contamination  
+  
+ - [ ] Data-based approach
+ - [ ] Model-based approach  
+
+Description of your method, 3-4 sentences. Evidence of data contamination (Read below):
+
+#### Data-based approaches  
+Data-based approaches identify evidence of data contamination in a pre-training corpus by directly examining the dataset for instances of the evaluation data. This method involves algorithmically searching through a large pre-training dataset to find occurrences of the evaluation data. You should provide evidence of data contamination in the form: "dataset X appears in line N of corpus Y," "dataset X appears N times in corpus Y," or "N examples from dataset X appear in corpus Y."   
+  
+#### Model-based approaches  
+  
+Model-based approaches, on the other hand, utilize heuristic algorithms to infer the presence of data contamination in a pre-trained model. These methods do not directly analyze the data but instead assess the model's behavior to predict data contamination. Examples include prompting the model to reproduce elements of an evaluation dataset to demonstrate memorization (i.e https://hitz-zentroa.github.io/lm-contamination/blog/), or using perplexity measures to estimate data contamination (). You should provide evidence of data contamination in the form of evaluation results of the algorithm from research papers, screenshots of model outputs that demonstrate memorization of a pre-training dataset, or any other form of evaluation that substantiates the method's effectiveness in detecting data contamination. You can provide a confidence score in your predictions.   
+
+## Citation  
+  
+Is there a paper that reports the data contamination or describes the method used to detect data contamination?  
+  
+URL: `https://aclanthology.org/2023.findings-emnlp.722/`
+Citation: `@inproceedings{...`
+```
+---
+
+### How to update the contamination_report.csv file
+
+The [contamination_report.csv](https://huggingface.co/spaces/CONDA-Workshop/Data-Contamination-Report/blob/main/contamination_report.csv) file is a csv filed with `;` delimiters. You will need to update the following columns:
+- Evaluation Dataset: Name of the evaluation dataset contaminated. If available in the HuggingFace Hub please write the path  (e.g. `uonlp/CulturaX`), otherwise  proviede the name of the dataset.
+- Contaminated Source: Name of the model that has been trained with the evaluation dataset or name of the pre-training copora that contains the evaluation datset. If available in the HuggingFace Hub please write the path  (e.g. `allenai/OLMo-7B`), otherwise proviede the name of the model/dataset.
+- Train split: Percentage of the train split contaminated. 0 means no contamination. 1 means that the dataset has been fully contamianted. If the dataset doesn't have splits, you can consider that the full dataset is a train or test split. 
+- Development split: Percentage of the development split contaminated. 0 means no contamination. 1 means that the dataset has been fully contamianted.
+- Train split: Percentage of the test split contaminated. 0 means no contamination. 1 means that the dataset has been fully contamianted.  If the dataset doesn't have splits, you can consider that the full dataset is a train or test split. 
+- Approach: data-based or model-based approach. See above for more information.
+- Citation: If there is paper or any other resource describing how you have detected this contamination example, provide the URL. 
+- PR Link: Leave it blank, we will update it after you create the Pull Request. 
+""".strip()
+
+
+PANEL_MARKDOWN = """
+# Data Contamination Report
+The 💨Data Contamination Report aims to track instances of data contamination in pre-trained models and corpora. 
+This effort is part of [The 1st Workshop on Data Contamination (CONDA)](https://conda-workshop.github.io/) that will be held at ACL 2024.
+""".strip()

requirements.txt

@@ -0,0 +1,7 @@
+gradio
+setuptools
+filelock
+pandas
+beautifulsoup4
+requests
+huggingface_hub

utils.py

@@ -0,0 +1,181 @@
+import logging
+import re
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from typing import Dict, List, Union
+from urllib.parse import urljoin, urlparse
+
+import requests
+from bs4 import BeautifulSoup
+
+
+def get_base_url(url: str) -> str:
+    """
+    Extracts the base URL from a given URL.
+
+    Parameters:
+    - url (str): The URL to extract the base URL from.
+
+    Returns:
+    - str: The base URL.
+    """
+    parsed_url = urlparse(url)
+    base_url = f"{parsed_url.scheme}://{parsed_url.netloc}"
+    return base_url
+
+
+def get_domain_name(url: str) -> str:
+    """
+    Get the domain name from a URL.
+
+    Args:
+        url (str): The URL.
+
+    Returns:
+        str: The domain name.
+    """
+
+    parsed_uri = urlparse(url)
+    domain = "{uri.netloc}".format(uri=parsed_uri)
+    if domain.startswith("www."):
+        domain = domain[4:]
+    # First latter in uppercase
+    return domain.capitalize()
+
+
+def get_favicon(url: str) -> str:
+    headers = {
+        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3"
+    }
+    try:
+        response = requests.get(url, headers=headers, timeout=2)
+        if response.status_code == 200:
+            soup = BeautifulSoup(response.content, "html.parser")
+            # Search for all potential icons including meta tags
+            icon_links = soup.find_all(
+                "link", rel=re.compile(r"(shortcut icon|icon|apple-touch-icon)", re.I)
+            )
+            meta_icons = soup.find_all(
+                "meta", attrs={"content": re.compile(r".ico$", re.I)}
+            )
+            icons = icon_links + meta_icons
+
+            if icons:
+                for icon in icons:
+                    favicon_url = icon.get("href") or icon.get("content")
+                    if favicon_url:
+                        if favicon_url.startswith("/"):
+                            favicon_url = urljoin(url, favicon_url)
+                        return favicon_url
+                # If icons found but no href or content, return default
+                return "https://upload.wikimedia.org/wikipedia/commons/0/01/Website_icon.svg"
+            else:
+                # No icons found, return default
+                return "https://upload.wikimedia.org/wikipedia/commons/0/01/Website_icon.svg"
+        else:
+            # Response was not OK, return default
+            return (
+                "https://upload.wikimedia.org/wikipedia/commons/0/01/Website_icon.svg"
+            )
+    except requests.Timeout:
+        logging.warning(f"Request timed out for {url}")
+        return "https://upload.wikimedia.org/wikipedia/commons/0/01/Website_icon.svg"
+    except Exception as e:
+        logging.warning(f"An error occurred while fetching favicon for {url}: {e}")
+        return "https://upload.wikimedia.org/wikipedia/commons/0/01/Website_icon.svg"
+
+
+def download_favicons(urls: List[str]) -> Dict[str, str]:
+    favicons = {}
+    urls = list(set(urls))
+    with ThreadPoolExecutor(max_workers=20) as executor:
+        future_to_url = {executor.submit(get_favicon, url): url for url in urls}
+        for future in as_completed(future_to_url):
+            url = future_to_url[future]
+            try:
+                favicon_url = future.result()
+                favicons[url] = favicon_url
+            except Exception as e:
+                logging.warning(f"Failed to fetch favicon for {url}: {e}")
+                favicons[url] = (
+                    "https://upload.wikimedia.org/wikipedia/commons/0/01/Website_icon.svg"
+                )
+    return favicons
+
+
+def url_exists(url):
+    """
+    Checks if a URL exists by making a HEAD request.
+
+    Parameters:
+    - url (str): The URL to check.
+
+    Returns:
+    - bool: True if the URL exists, False otherwise.
+    """
+    try:
+        response = requests.head(url, allow_redirects=True)
+        return response.status_code < 400
+    except requests.RequestException:
+        # In case of network problems, SSL errors, etc.
+        return False
+
+
+def build_dataset_url(dataset_name: str):
+    """
+    Build an HTML string with the dataset URL.
+    """
+    url = f"https://huggingface.co/datasets/{dataset_name}"
+    # Test if the url exists
+    if url_exists(url):
+        return url
+    else:
+        return None
+
+
+def build_model_url(model_name: str):
+    """
+    Build an HTML string with the model URL.
+    """
+    url = f"https://huggingface.co/{model_name}"
+    # Test if the url exists
+    if url_exists(url):
+        return url
+    else:
+        return None
+
+
+def build_text_icon(text: str, url: Union[str, None], icon_url: str):
+    if url is not None:
+        return (
+            f'<a href="{url}" target="_blank" style="text-decoration: none; color: inherit; display: inline-flex; align-items: center;">'
+            f'<img src="{icon_url}" alt="{url}" style="display: inline-block; vertical-align: middle; margin-right: 4px;" width="16" height="16">'
+            f'<span style="display: inline-block; vertical-align: middle;">{text}</span> </a>'
+        )
+    else:
+        return text
+
+
+def build_datasets_urls(datasets_names: List[str]) -> Dict[str, str]:
+    """
+    Build a dictionary of dataset URLs from a list of dataset names.
+
+    Parameters:
+    - datasets_names (List[str]): The list of dataset names.
+
+    Returns:
+    - Dict[str, str]: A dictionary of dataset URLs.
+    """
+    return {dataset: build_dataset_url(dataset) for dataset in datasets_names}
+
+
+def build_models_urls(models_names: List[str]) -> Dict[str, str]:
+    """
+    Build a dictionary of model URLs from a list of model names.
+
+    Parameters:
+    - models_names (List[str]): The list of model names.
+
+    Returns:
+    - Dict[str, str]: A dictionary of model URLs.
+    """
+    return {model: build_model_url(model) for model in models_names}