WIP: Dynamic block-based results, funky reporting

app.py

@@ -1,7 +1,9 @@
 import gradio as gr
+from huggingface_hub import ModelCard
 
 from compliance_checks import (
     ComplianceSuite,
+    ComplianceCheck,
     ModelProviderIdentityCheck,
     IntendedPurposeCheck,
     GeneralLimitationsCheck,
@@ -10,42 +12,83 @@ from compliance_checks import (
 
 from bloom_card import bloom_card
 
+checks = [
+    ModelProviderIdentityCheck(),
+    IntendedPurposeCheck(),
+    GeneralLimitationsCheck(),
+    ComputationalRequirementsCheck(),
+]
+suite = ComplianceSuite(checks=checks)
+
+
+def status_emoji(status: bool):
+    return "✅" if status else "🛑"
 
-def run_compliance_check(model_card: str):
-    suite = ComplianceSuite(checks=[
-        ModelProviderIdentityCheck(),
-        IntendedPurposeCheck(),
-        GeneralLimitationsCheck(),
-        ComputationalRequirementsCheck(),
-    ])
 
+def run_compliance_check(model_card: str):
     results = suite.run(model_card)
 
-    return str([r[0] for r in results])
+    return [
+        *[gr.Accordion.update(label=f"{r.name} - {status_emoji(r.status)}") for r in results],
+        *[gr.Markdown.update(value=r.to_string()) for r in results],
+    ]
+
+
+def fetch_and_run_compliance_check(model_id: str):
+    model_card = ModelCard.load(repo_id_or_path=model_id).content
+    return run_compliance_check(model_card=model_card)
 
 
-with gr.Blocks() as demo:
+def compliance_result(compliance_check: ComplianceCheck):
+    accordion = gr.Accordion(label=f"{compliance_check.name}", open=False)
+    with accordion:
+        description = gr.Markdown("Run an evaluation to see results...")
+
+    return accordion, description
+
+
+with gr.Blocks(css="#reverse-row { flex-direction: row-reverse; }") as demo:
     gr.Markdown("""\
     # Model Card Validator
     Following Article 13 of the EU AI Act
     """)
 
-    with gr.Row():
-        with gr.Column():
-            model_card_box = gr.TextArea()
-            populate_sample = gr.Button(value="Populate Sample")
-            submit = gr.Button()
+    with gr.Row(elem_id="reverse-row"):
+        with gr.Tab(label="Results"):
+            with gr.Column():
+                compliance_results = [compliance_result(c) for c in suite.checks]
+                compliance_accordions = [c[0] for c in compliance_results]
+                compliance_descriptions = [c[1] for c in compliance_results]
 
         with gr.Column():
-            results_list = gr.Text()
+            with gr.Tab(label="Markdown"):
+                model_card_box = gr.TextArea()
+                populate_sample_card = gr.Button(value="Populate Sample")
+                submit_markdown = gr.Button()
+            with gr.Tab(label="Search for Model"):
+                model_id_search = gr.Text()
+                submit_model_search = gr.Button()
+                gr.Examples(
+                    examples=["society-ethics/model-card-webhook-test"],
+                    inputs=[model_id_search],
+                    outputs=[*compliance_accordions, *compliance_descriptions],
+                    fn=fetch_and_run_compliance_check,
+                    # cache_examples=True,  # TODO: Why does this break the app?
+                )
 
-    submit.click(
+    submit_markdown.click(
         fn=run_compliance_check,
         inputs=[model_card_box],
-        outputs=[results_list]
+        outputs=[*compliance_accordions, *compliance_descriptions]
+    )
+
+    submit_model_search.click(
+        fn=fetch_and_run_compliance_check,
+        inputs=[model_id_search],
+        outputs=[*compliance_accordions, *compliance_descriptions]
     )
 
-    populate_sample.click(
+    populate_sample_card.click(
         fn=lambda: bloom_card,
         inputs=[],
         outputs=[model_card_box]

compliance_checks.py

@@ -1,30 +1,10 @@
 from abc import ABC, abstractmethod
+from typing import Optional, List
 
 import markdown
 from bs4 import BeautifulSoup, Comment
 
 
-class ComplianceCheck(ABC):
-    @abstractmethod
-    def run_check(self, card: BeautifulSoup):
-        raise NotImplementedError
-
-
-class ModelProviderIdentityCheck(ComplianceCheck):
-    def run_check(self, card: BeautifulSoup):
-        try:
-            developed_by = card.find("strong", string="Developed by:")
-
-            developer = "".join([str(s) for s in developed_by.next_siblings]).strip()
-
-            if developer == "[More Information Needed]":
-                return False, None
-
-            return True, developer
-        except AttributeError:
-            return False, None
-
-
 def walk_to_next_heading(card, heading, heading_text):
     stop_at = [heading, f"h{int(heading[1]) - 1}"]
 
@@ -49,33 +29,203 @@ def walk_to_next_heading(card, heading, heading_text):
         return False, None
 
 
+class ComplianceResult(ABC):
+    name: str = None
+
+    def __init__(self, status: Optional[bool] = False, *args, **kwargs):
+        self.status = status
+
+    def __eq__(self, other):
+        try:
+            assert self.status == other.status
+            return True
+        except AssertionError:
+            return False
+
+    @abstractmethod
+    def to_string(self):
+        return "Not Implemented"
+
+
+class ComplianceCheck(ABC):
+    name: str = None
+
+    @abstractmethod
+    def run_check(self, card: BeautifulSoup) -> ComplianceResult:
+        raise NotImplementedError
+
+
+class ModelProviderIdentityResult(ComplianceResult):
+    name = "Model Provider Identity"
+
+    def __init__(self, provider: str = None, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.provider = provider
+
+    def __eq__(self, other):
+        if isinstance(other, ModelProviderIdentityResult):
+            if super().__eq__(other):
+                try:
+                    assert self.provider == other.provider
+                    return True
+                except AssertionError:
+                    return False
+        else:
+            return False
+
+    def to_string(self):
+        return str(self.provider)
+
+
+class ModelProviderIdentityCheck(ComplianceCheck):
+    name = "Model Provider Identity"
+
+    def run_check(self, card: BeautifulSoup):
+        try:
+            developed_by = card.find("strong", string="Developed by:")
+
+            developer = "".join([str(s) for s in developed_by.next_siblings]).strip()
+
+            if developer == "[More Information Needed]":
+                return ModelProviderIdentityResult()
+
+            return ModelProviderIdentityResult(status=True, provider=developer)
+        except AttributeError:
+            return ModelProviderIdentityResult()
+
+
+class IntendedPurposeResult(ComplianceResult):
+    name = "Intended Purpose"
+
+    def __init__(
+            self,
+            direct_use: str = None,
+            downstream_use: str = None,
+            out_of_scope_use: str = None,
+            *args,
+            **kwargs,
+    ):
+        super().__init__(*args, **kwargs)
+        self.direct_use = direct_use
+        self.downstream_use = downstream_use
+        self.out_of_scope_use = out_of_scope_use
+
+    def __eq__(self, other):
+        if isinstance(other, IntendedPurposeResult):
+            if super().__eq__(other):
+                try:
+                    assert self.direct_use == other.direct_use
+                    assert self.downstream_use == other.downstream_use
+                    assert self.out_of_scope_use == other.out_of_scope_use
+                    return True
+                except AssertionError:
+                    return False
+        else:
+            return False
+
+    def to_string(self):
+        return str((self.direct_use, self.direct_use, self.out_of_scope_use))
+
+
 class IntendedPurposeCheck(ComplianceCheck):
+    name = "Intended Purpose"
+
     def run_check(self, card: BeautifulSoup):
         direct_use_check, direct_use_content = walk_to_next_heading(card, "h3", "Direct Use")
         # TODO: Handle [optional], which doesn't exist in BLOOM, e.g.
         downstream_use_check, downstream_use_content = walk_to_next_heading(card, "h3", "Downstream Use [optional]")
         out_of_scope_use_check, out_of_scope_use_content = walk_to_next_heading(card, "h3", "Out-of-Scope Use")
-        return (
-            direct_use_check and out_of_scope_use_check,
-            [direct_use_content, downstream_use_content, out_of_scope_use_content]
+        return IntendedPurposeResult(
+            status=direct_use_check and out_of_scope_use_check,
+            direct_use=direct_use_content,
+            downstream_use=downstream_use_content,
+            out_of_scope_use=out_of_scope_use_content
         )
 
 
+class GeneralLimitationsResult(ComplianceResult):
+    name = "General Limitations"
+
+    def __init__(
+            self,
+            limitations: str = None,
+            *args,
+            **kwargs,
+    ):
+        super().__init__(*args, **kwargs)
+        self.limitations = limitations
+
+    def __eq__(self, other):
+        if isinstance(other, GeneralLimitationsResult):
+            if super().__eq__(other):
+                try:
+                    assert self.limitations == other.limitations
+                    return True
+                except AssertionError:
+                    return False
+        else:
+            return False
+
+    def to_string(self):
+        return self.limitations
+
+
 class GeneralLimitationsCheck(ComplianceCheck):
+    name = "General Limitations"
+
     def run_check(self, card: BeautifulSoup):
-        return walk_to_next_heading(card, "h2", "Bias, Risks, and Limitations")
+        check, content = walk_to_next_heading(card, "h2", "Bias, Risks, and Limitations")
+
+        return GeneralLimitationsResult(
+            status=check,
+            limitations=content
+        )
+
+
+class ComputationalRequirementsResult(ComplianceResult):
+    name = "Computational Requirements"
+
+    def __init__(
+            self,
+            requirements: str = None,
+            *args,
+            **kwargs,
+    ):
+        super().__init__(*args, **kwargs)
+        self.requirements = requirements
+
+    def __eq__(self, other):
+        if isinstance(other, ComputationalRequirementsResult):
+            if super().__eq__(other):
+                try:
+                    assert self.requirements == other.requirements
+                    return True
+                except AssertionError:
+                    return False
+        else:
+            return False
+
+    def to_string(self):
+        return self.requirements
 
 
 class ComputationalRequirementsCheck(ComplianceCheck):
+    name = "Computational Requirements"
+
     def run_check(self, card: BeautifulSoup):
-        return walk_to_next_heading(card, "h3", "Compute infrastructure")
+        check, content = walk_to_next_heading(card, "h3", "Compute infrastructure")
+
+        return ComputationalRequirementsResult(
+            status=check,
+            requirements=content,
+        )
 
 
 class ComplianceSuite:
     def __init__(self, checks):
         self.checks = checks
 
-    def run(self, model_card):
+    def run(self, model_card) -> List[ComplianceResult]:
         model_card_html = markdown.markdown(model_card)
         card_soup = BeautifulSoup(model_card_html, features="html.parser")
 

tests/test_compliance_checks.py

@@ -5,10 +5,10 @@ import markdown
 from bs4 import BeautifulSoup
 from compliance_checks import (
     ComplianceSuite,
-    ModelProviderIdentityCheck,
-    IntendedPurposeCheck,
-    GeneralLimitationsCheck,
-    ComputationalRequirementsCheck,
+    ModelProviderIdentityCheck, ModelProviderIdentityResult,
+    IntendedPurposeCheck, IntendedPurposeResult,
+    GeneralLimitationsCheck, GeneralLimitationsResult,
+    ComputationalRequirementsCheck, ComputationalRequirementsResult,
 )
 
 
@@ -201,26 +201,39 @@ Etc..
 Etc..
 """
 
-    @pytest.mark.parametrize("check, card,check_passed,values", [
-        (ModelProviderIdentityCheck(), "provider_identity_model_card", True, "Nima Boscarino"),
-        (ModelProviderIdentityCheck(), "bad_provider_identity_model_card", False, None),
-        (IntendedPurposeCheck(), "intended_purpose_model_card", True, ["Here is some info about direct uses...", None, "Here is some info about out-of-scope uses..."]),
-        (IntendedPurposeCheck(), "bad_intended_purpose_model_card", False, [None, None, None]),
-        (GeneralLimitationsCheck(), "general_limitations_model_card", True, "Hello world! These are some risks..."),
-        (GeneralLimitationsCheck(), "bad_general_limitations_model_card", False, None),
-        (ComputationalRequirementsCheck(), "computational_requirements_model_card", True, expected_infrastructure),
-        (ComputationalRequirementsCheck(), "bad_computational_requirements_model_card", False, None),
+    @pytest.mark.parametrize("check,card,expected", [
+        (ModelProviderIdentityCheck(), "provider_identity_model_card", ModelProviderIdentityResult(
+            status=True,
+            provider="Nima Boscarino",
+        )),
+        (ModelProviderIdentityCheck(), "bad_provider_identity_model_card", ModelProviderIdentityResult()),
+        (IntendedPurposeCheck(), "intended_purpose_model_card", IntendedPurposeResult(
+            status=True,
+            direct_use="Here is some info about direct uses...",
+            downstream_use=None,
+            out_of_scope_use="Here is some info about out-of-scope uses...",
+        )),
+        (IntendedPurposeCheck(), "bad_intended_purpose_model_card", IntendedPurposeResult()),
+        (GeneralLimitationsCheck(), "general_limitations_model_card", GeneralLimitationsResult(
+            status=True,
+            limitations="Hello world! These are some risks..."
+        )),
+        (GeneralLimitationsCheck(), "bad_general_limitations_model_card", GeneralLimitationsResult()),
+        (ComputationalRequirementsCheck(), "computational_requirements_model_card", ComputationalRequirementsResult(
+            status=True,
+            requirements=expected_infrastructure,
+        )),
+        (ComputationalRequirementsCheck(), "bad_computational_requirements_model_card", ComputationalRequirementsResult()),
     ])
-    def test_run_model_provider_identity_check(self, check, card, check_passed, values, request):
+    def test_run_checks(self, check, card, expected, request):
         card = request.getfixturevalue(card)
 
         model_card_html = markdown.markdown(card)
         card_soup = BeautifulSoup(model_card_html, features="html.parser")
 
-        results_check_passed, results_values = check.run_check(card_soup)
+        results = check.run_check(card_soup)
 
-        assert results_check_passed == check_passed
-        assert results_values == values
+        assert results == expected
 
 
 class TestComplianceSuite:
@@ -333,4 +346,4 @@ Jean Zay Public Supercomputer, provided by the French government.
 
         results = suite.run(card)
 
-        assert all([r[0] for r in results])
+        assert all([r.status for r in results])