fixed checkbox logic to allow multiple selections

app.py

@@ -4,13 +4,31 @@ import pandas as pd
 from gradio_modal import Modal
 import os
 import yaml
-
+import itertools
 
 folder_path = 'configs'
 # List to store data from YAML files
 data_list = []
 metadata_dict = {}
 
+
+def expand_string_list(string_list):
+    expanded_list = []
+
+    # Add individual strings to the expanded list
+    expanded_list.extend(string_list)
+
+    # Generate combinations of different lengths from the input list
+    for r in range(2, len(string_list) + 1):
+        combinations = itertools.combinations(string_list, r)
+        for combination in combinations:
+            # Generate permutations of each combination
+            permutations = itertools.permutations(combination)
+            for permutation in permutations:
+                expanded_list.append(' + '.join(permutation))
+
+    return expanded_list
+
 # Iterate over each file in the folder
 for filename in os.listdir(folder_path):
     if filename.endswith('.yaml'):
@@ -27,11 +45,14 @@ globaldf['Link'] = '<u>'+globaldf['Link']+'</u>'
 
 # Define the desired order of categories
 modality_order = ["Text", "Image", "Audio", "Video"]
-type_order = ["Model", "Dataset", "Output", "Taxonomy"]
+level_order = ["Model", "Dataset", "Output", "Taxonomy"]
+
+modality_order = expand_string_list(modality_order)
+level_order = expand_string_list(level_order)
 
 # Convert Modality and Level columns to categorical with specified order
 globaldf['Modality'] = pd.Categorical(globaldf['Modality'], categories=modality_order, ordered=True)
-globaldf['Level'] = pd.Categorical(globaldf['Level'], categories=type_order, ordered=True)
+globaldf['Level'] = pd.Categorical(globaldf['Level'], categories=level_order, ordered=True)
 
 # Sort DataFrame by Modality and Level
 globaldf.sort_values(by=['Modality', 'Level'], inplace=True)
@@ -40,8 +61,8 @@ globaldf.sort_values(by=['Modality', 'Level'], inplace=True)
 
 # Path: taxonomy.py
 
-def filter_modality_type(fulltable, modality_filter, type_filter):
-    filteredtable = fulltable[fulltable['Modality'].isin(modality_filter) & fulltable['Level'].isin(type_filter)]
+def filter_modality_level(fulltable, modality_filter, level_filter):
+    filteredtable = fulltable[fulltable['Modality'].str.contains('|'.join(modality_filter)) & fulltable['Level'].str.contains('|'.join(level_filter))]
     return filteredtable
 
 def showmodal(evt: gr.SelectData):
@@ -92,18 +113,18 @@ with gr.Blocks(title = "Social Impact Measurement V2", css=custom_css) as demo:
         gr.Markdown("""
 #### Technical Base System Evaluations:
                     
-Below we list the aspects possible to evaluate in a generative system. Context-absent evaluations only provide narrow insights into the described aspects of the type of generative AI system. The depth of literature and research on evaluations differ by modality with some modalities having sparse or no relevant literature, but the themes for evaluations can be applied to most systems.
+Below we list the aspects possible to evaluate in a generative system. Context-absent evaluations only provide narrow insights into the described aspects of the level of generative AI system. The depth of literature and research on evaluations differ by modality with some modalities having sparse or no relevant literature, but the themes for evaluations can be applied to most systems.
 
 The following categories are high-level, non-exhaustive, and present a synthesis of the findings across different modalities. They refer solely to what can be evaluated in a base technical system:
 
                     """)
     with gr.Tabs(elem_classes="tab-buttons") as tabs1:
-        with gr.TabItem("Bias/Stereotypes"):
+        with gr.TabItem("Bias/Stereolevels"):
             fulltable = globaldf[globaldf['Group'] == 'BiasEvals']
             fulltable = fulltable[['Modality','Level', 'Suggested Evaluation', 'What it is evaluating', 'Considerations', 'Link']]
 
             gr.Markdown("""
-            Generative AI systems can perpetuate harmful biases from various sources, including systemic, human, and statistical biases. These biases, also known as "fairness" considerations, can manifest in the final system due to choices made throughout the development process. They include harmful associations and stereotypes related to protected classes, such as race, gender, and sexuality. Evaluating biases involves assessing correlations, co-occurrences, sentiment, and toxicity across different modalities, both within the model itself and in the outputs of downstream tasks.
+            Generative AI systems can perpetuate harmful biases from various sources, including systemic, human, and statistical biases. These biases, also known as "fairness" considerations, can manifest in the final system due to choices made throughout the development process. They include harmful associations and stereolevels related to protected classes, such as race, gender, and sexuality. Evaluating biases involves assessing correlations, co-occurrences, sentiment, and toxicity across different modalities, both within the model itself and in the outputs of downstream tasks.
                         """)
             with gr.Row():
                 modality_filter = gr.CheckboxGroup(["Text", "Image", "Audio", "Video"], 
@@ -112,7 +133,7 @@ The following categories are high-level, non-exhaustive, and present a synthesis
                                                  show_label=True,
                                                 #  info="Which modality to show."
                                                  )
-                type_filter = gr.CheckboxGroup(["Model", "Dataset", "Output", "Taxonomy"], 
+                level_filter = gr.CheckboxGroup(["Model", "Dataset", "Output", "Taxonomy"], 
                                                  value=["Model", "Dataset", "Output", "Taxonomy"], 
                                                  label="Level", 
                                                  show_label=True,
@@ -121,8 +142,8 @@ The following categories are high-level, non-exhaustive, and present a synthesis
             with gr.Row():
                 table_full = gr.DataFrame(value=fulltable, wrap=True, datatype="markdown", visible=False, interactive=False)
                 table_filtered = gr.DataFrame(value=fulltable, wrap=True, datatype="markdown", visible=True, interactive=False)
-                modality_filter.change(filter_modality_type, inputs=[table_full, modality_filter, type_filter], outputs=table_filtered)
-                type_filter.change(filter_modality_type, inputs=[table_full, modality_filter, type_filter], outputs=table_filtered)
+                modality_filter.change(filter_modality_level, inputs=[table_full, modality_filter, level_filter], outputs=table_filtered)
+                level_filter.change(filter_modality_level, inputs=[table_full, modality_filter, level_filter], outputs=table_filtered)
 
 
                 with Modal(visible=False) as modal:
@@ -150,7 +171,7 @@ The following categories are high-level, non-exhaustive, and present a synthesis
                                                  show_label=True,
                                                 #  info="Which modality to show."
                                                  )
-                type_filter = gr.CheckboxGroup(["Model", "Dataset", "Output", "Taxonomy"], 
+                level_filter = gr.CheckboxGroup(["Model", "Dataset", "Output", "Taxonomy"], 
                                                  value=["Model", "Dataset", "Output", "Taxonomy"], 
                                                  label="Level", 
                                                  show_label=True,
@@ -159,8 +180,8 @@ The following categories are high-level, non-exhaustive, and present a synthesis
             with gr.Row():
                 table_full = gr.DataFrame(value=fulltable, wrap=True, datatype="markdown", visible=False, interactive=False)
                 table_filtered = gr.DataFrame(value=fulltable, wrap=True, datatype="markdown", visible=True, interactive=False)
-                modality_filter.change(filter_modality_type, inputs=[table_full, modality_filter, type_filter], outputs=table_filtered)
-                type_filter.change(filter_modality_type, inputs=[table_full, modality_filter, type_filter], outputs=table_filtered)
+                modality_filter.change(filter_modality_level, inputs=[table_full, modality_filter, level_filter], outputs=table_filtered)
+                level_filter.change(filter_modality_level, inputs=[table_full, modality_filter, level_filter], outputs=table_filtered)
 
 
                 with Modal(visible=False) as modal:
@@ -179,8 +200,40 @@ The following categories are high-level, non-exhaustive, and present a synthesis
         #         gr.Image()
 
         with gr.TabItem("Privacy/Data Protection"):
+            fulltable = globaldf[globaldf['Group'] == 'PrivacyEvals']
+            fulltable = fulltable[['Modality','Level', 'Suggested Evaluation', 'What it is evaluating', 'Considerations', 'Link']]
+
+            gr.Markdown("""Cultural values are specific to groups and sensitive content is normative. Sensitive topics also vary by culture and can include hate speech. What is considered a sensitive topic, such as egregious violence or adult sexual content, can vary widely by viewpoint. Due to norms differing by culture, region, and language, there is no standard for what constitutes sensitive content.
+                        Distinct cultural values present a challenge for deploying models into a global sphere, as what may be appropriate in one culture may be unsafe in others. Generative AI systems cannot be neutral or objective, nor can they encompass truly universal values. There is no “view from nowhere”; in quantifying anything, a particular frame of reference is imposed.
+                        """)
             with gr.Row():
-                gr.Image()
+                modality_filter = gr.CheckboxGroup(["Text", "Image", "Audio", "Video"], 
+                                                 value=["Text", "Image", "Audio", "Video"], 
+                                                 label="Modality", 
+                                                 show_label=True,
+                                                #  info="Which modality to show."
+                                                 )
+                level_filter = gr.CheckboxGroup(["Model", "Dataset", "Output", "Taxonomy"], 
+                                                 value=["Model", "Dataset", "Output", "Taxonomy"], 
+                                                 label="Level", 
+                                                 show_label=True,
+                                                #  info="Which modality to show."
+                                                 )
+            with gr.Row():
+                table_full = gr.DataFrame(value=fulltable, wrap=True, datatype="markdown", visible=False, interactive=False)
+                table_filtered = gr.DataFrame(value=fulltable, wrap=True, datatype="markdown", visible=True, interactive=False)
+                modality_filter.change(filter_modality_level, inputs=[table_full, modality_filter, level_filter], outputs=table_filtered)
+                level_filter.change(filter_modality_level, inputs=[table_full, modality_filter, level_filter], outputs=table_filtered)
+
+
+                with Modal(visible=False) as modal:
+                    titlemd = gr.Markdown(visible=False)
+                    authormd = gr.Markdown(visible=False)
+                    tagsmd = gr.Markdown(visible=False)
+                    abstractmd = gr.Markdown(visible=False)
+                    datasetmd = gr.Markdown(visible=False)
+                    gallery = gr.Gallery(visible=False)
+                table_filtered.select(showmodal, None, [modal, titlemd, authormd, tagsmd, abstractmd, datasetmd, gallery])
 
         # with gr.TabItem("Financial Costs"):
         #     with gr.Row():

configs/measuringforgetting.yaml

@@ -0,0 +1,19 @@
+Abstract: "Machine learning models exhibit two seemingly contradictory phenomena: training data memorization, and various forms of forgetting. In memorization, models overfit specific training examples and become susceptible to privacy attacks. In forgetting, examples which appeared early in training are forgotten by the end. In this work, we connect these phenomena. We propose a technique to measure to what extent models \"forget\" the specifics of training examples, becoming less susceptible to privacy attacks on examples they have not seen recently. We show that, while non-convex models can memorize data forever in the worst-case, standard image, speech, and language models empirically do forget examples over time. We identify nondeterminism as a potential explanation, showing that deterministically trained models do not forget. Our results suggest that examples seen early when training with extremely large datasets - for instance those examples used to pre-train a model - may observe privacy benefits at the expense of examples seen later."
+Applicable Models:
+- ResNet (Image)
+- Conformer (Audio)
+- T5 (Text)
+Authors: Matthew Jagielski, Om Thakkar, Florian Tramèr, Daphne Ippolito, Katherine Lee, Nicholas Carlini, Eric Wallace, Shuang Song, Abhradeep Thakurta, Nicolas Papernot, Chiyuan Zhang
+Considerations: .nan
+Datasets: .nan
+Group: PrivacyEvals
+Hashtags: .nan
+Link: 'Measuring Forgetting of Memorized Training Examples'
+Modality: Text + Image + Audio
+Screenshots:
+- Images/Forgetting1.png
+- Images/Forgetting2.png
+Suggested Evaluation: Measuring forgetting of training examples
+Level: Model
+URL: https://arxiv.org/pdf/2207.00099.pdf
+What it is evaluating: Measure whether models forget training examples over time, over different types of models (image, audio, text) and how order of training affects privacy attacks