tests/test_compliance_checks.py

import pytest
from unittest.mock import MagicMock

from compliance_checks import (
    ComplianceSuite,
    IntendedPurposeCheck,
    GeneralLimitationsCheck,
)


class TestComplianceSuite:
    @pytest.fixture
    def mock_compliance_check(self):
        mockComplianceCheck = MagicMock()
        mockComplianceCheck.run_check = MagicMock(return_value=True)

        return mockComplianceCheck

    @pytest.fixture
    def empty_compliance_suite(self):
        return ComplianceSuite(
            checks=[]
        )

    @pytest.fixture
    def compliance_suite(self, mock_compliance_check):
        return ComplianceSuite(
            checks=[mock_compliance_check]
        )

    @pytest.fixture
    def empty_compliance_results(self):
        return []

    @pytest.fixture
    def compliance_results(self):
        return [True]

    def test_create_empty_compliance_suite(self, empty_compliance_suite):
        assert len(empty_compliance_suite.checks) == 0

    def test_create_compliance_suite(self, compliance_suite):
        assert len(compliance_suite.checks) == 1

    @pytest.mark.parametrize("suite,results", [
        ("empty_compliance_suite", "empty_compliance_results"),
        ("compliance_suite", "compliance_results")
    ])
    def test_run_compliance_suite(self, suite, results, request):
        suite: ComplianceSuite = request.getfixturevalue(suite)
        results: list = request.getfixturevalue(results)
        assert suite.run("") == results

        for check in suite.checks:
            check.run_check.assert_called_once()


def test_end_to_end_compliance_suite(real_model_card, expected_check_results):
    # TODO: TEMP
    expected_check_results = expected_check_results[:2]

    suite = ComplianceSuite(checks=[
        IntendedPurposeCheck(),
        GeneralLimitationsCheck(),
        # ComputationalRequirementsCheck()
    ])

    results = suite.run(real_model_card)

    assert all([r.status == e for r, e in zip(results, expected_check_results)])