WIP: Compliance Check pipeline w/ gradio app

app.py

@@ -0,0 +1,27 @@
+import gradio as gr
+from huggingface_hub import ModelCard
+from compliance_checks import (
+    ComplianceSuite,
+    ModelProviderIdentityCheck,
+    IntendedPurposeCheck
+)
+
+def run_compliance_check(repo_name):
+    model_card = ModelCard.load(repo_id_or_path=repo_name).content
+
+    suite = ComplianceSuite(checks=[
+        ModelProviderIdentityCheck(),
+        IntendedPurposeCheck()
+    ])
+
+    results = suite.run(model_card)
+
+    return str(results)
+
+
+gr.Interface(
+    fn=run_compliance_check,
+    inputs="text",
+    outputs="text",
+    examples=[["society-ethics/model-card-webhook-test"]]
+).launch()

compliance_checks.py

@@ -0,0 +1,59 @@
+from abc import ABC, abstractmethod
+
+import markdown
+from bs4 import BeautifulSoup, Comment
+
+
+class ComplianceCheck(ABC):
+    @abstractmethod
+    def run_check(self, card: BeautifulSoup) -> bool:
+        raise NotImplementedError
+
+
+class ModelProviderIdentityCheck(ComplianceCheck):
+    def run_check(self, card: BeautifulSoup):
+        try:
+            model_description = card.find("h3", string="Model Description")
+            description_list = model_description.find_next_siblings()[0]
+            developer = description_list.find(string="Developed by:").parent.next_sibling.strip()
+
+            if developer == "[More Information Needed]":
+                return False, None
+
+            return True, developer
+        except AttributeError:
+            return False, None
+
+
+class IntendedPurposeCheck(ComplianceCheck):
+    def run_check(self, card: BeautifulSoup):
+        try:
+            direct_use = card.find("h3", string="Direct Use")
+
+            direct_use_content = ""
+
+            sibling_gen = direct_use.nextSiblingGenerator()
+            sibling = next(sibling_gen)
+
+            while sibling.name != "h3":
+                if not isinstance(sibling, Comment):
+                    direct_use_content = direct_use_content + sibling.text
+                sibling = next(sibling_gen)
+
+            if direct_use_content.strip() == "[More Information Needed]":
+                return False, None
+
+            return True, None
+        except AttributeError:
+            return False, None
+
+
+class ComplianceSuite:
+    def __init__(self, checks):
+        self.checks = checks
+
+    def run(self, model_card):
+        model_card_html = markdown.markdown(model_card)
+        card_soup = BeautifulSoup(model_card_html, features="html.parser")
+
+        return [c.run_check(card_soup) for c in self.checks]

main.py

@@ -1,13 +1,8 @@
 import os
-from typing import Dict, Any, Optional, List
-import re
-from abc import ABC, abstractmethod
 
-from huggingface_hub import (ModelCard, comment_discussion,
+from huggingface_hub import (comment_discussion,
                              create_discussion, get_discussion_details,
                              get_repo_discussions)
-import markdown
-from bs4 import BeautifulSoup
 from tabulate import tabulate
 from difflib import SequenceMatcher
 
@@ -19,77 +14,6 @@ def similar(a, b):
     return SequenceMatcher(None, a, b).ratio()
 
 
-class ComplianceCheck(ABC):
-    def __init__(self, name):
-        self.name = name
-
-    @abstractmethod
-    def check(self, card: BeautifulSoup) -> bool:
-        raise NotImplementedError
-
-
-class ModelProviderIdentityCheck(ComplianceCheck):
-    def __init__(self):
-        super().__init__("Identity and Contact Details")
-
-    def check(self, card: BeautifulSoup):
-        developed_by_li = card.findAll(text=re.compile("Developed by"))[0].parent.parent
-        developed_by = list(developed_by_li.children)[1].text.strip()
-
-        if developed_by == "[More Information Needed]":
-            return False
-        else:
-            return True
-
-
-class IntendedPurposeCheck(ComplianceCheck):
-    def __init__(self):
-        super().__init__("Intended Purpose")
-
-    def check(self, card: BeautifulSoup):
-
-        # direct_use = card.find_all("h2", text="Direct Use")[0]
-        #
-        # if developed_by == "[More Information Needed]":
-        #     return False
-        # else:
-        return False
-
-
-compliance_checks = [
-    ModelProviderIdentityCheck(),
-    IntendedPurposeCheck()
-    # "General Limitations",
-    # "Computational and Hardware Requirements",
-    # "Carbon Emissions"
-]
-
-
-def parse_webhook_post(data: Dict[str, Any]) -> Optional[str]:
-    event = data["event"]
-    if event["scope"] != "repo":
-        return None
-    repo = data["repo"]
-    repo_name = repo["name"]
-    repo_type = repo["type"]
-    if repo_type != "model":
-        raise ValueError("Incorrect repo type.")
-    return repo_name
-
-
-def check_compliance(comp_checks: List[ComplianceCheck], card: BeautifulSoup) -> Dict[str, bool]:
-    return {c.name: c.check(card) for c in comp_checks}
-
-
-def run_compliance_check(repo_name):
-    card_data: ModelCard = ModelCard.load(repo_id_or_path=repo_name)
-    card_html = markdown.markdown(card_data.content)
-    card_soup = BeautifulSoup(card_html, features="html.parser")
-    compliance_results = check_compliance(compliance_checks, card_soup)
-
-    return compliance_results
-
-
 def create_metadata_breakdown_table(compliance_check_dictionary):
     data = {k: v for k, v in compliance_check_dictionary.items()}
     metadata_fields_column = list(data.keys())

requirements.txt

@@ -1,5 +1,7 @@
-fastapi
-uvicorn
-markdown
-beautifulsoup4
-tabulate
+# fastapi
+# uvicorn
+# markdown
+# beautifulsoup4
+# tabulate
+# pytest
+gradio

tests/test_compliance_checks.py

@@ -0,0 +1,200 @@
+import pytest
+from unittest.mock import MagicMock
+
+import markdown
+from bs4 import BeautifulSoup, Comment
+from compliance_checks import ComplianceSuite, ModelProviderIdentityCheck, IntendedPurposeCheck
+
+
+class TestComplianceCheck:
+    @pytest.fixture
+    def provider_identity_model_card(self):
+        return """
+# Model Card for Sample Model
+
+Some random info...
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+- **Developed by:** Nima Boscarino
+- **Model type:** Yada yada yada
+        """
+
+    @pytest.fixture
+    def bad_provider_identity_model_card(self):
+        return """
+# Model Card for Sample Model
+
+Some random info...
+
+## Model Details
+
+### Model Description
+
+- **Developed by:** [More Information Needed]
+- **Model type:** Yada yada yada
+        """
+
+    @pytest.fixture
+    def intended_purpose_model_card(self):
+        return """
+# Model Card for Sample Model
+
+Some random info...
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+Here is some info about direct uses...
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]        
+        """
+
+    @pytest.fixture
+    def bad_intended_purpose_model_card(self):
+        return """
+# Model Card for Sample Model
+
+Some random info...
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+        """
+
+    @pytest.mark.parametrize("check, card,check_passed,values", [
+        (ModelProviderIdentityCheck(), "provider_identity_model_card", True, "Nima Boscarino"),
+        (ModelProviderIdentityCheck(), "bad_provider_identity_model_card", False, None),
+        (IntendedPurposeCheck(), "intended_purpose_model_card", True, None),
+        (IntendedPurposeCheck(), "bad_intended_purpose_model_card", False, None),
+    ])
+    def test_run_model_provider_identity_check(self, check, card, check_passed, values, request):
+        card = request.getfixturevalue(card)
+
+        model_card_html = markdown.markdown(card)
+        card_soup = BeautifulSoup(model_card_html, features="html.parser")
+
+        results_check_passed, results_values = check.run_check(card_soup)
+
+        assert results_check_passed == check_passed
+        assert results_values == values
+
+
+class TestComplianceSuite:
+    @pytest.fixture
+    def mock_compliance_check(self):
+        mockComplianceCheck = MagicMock()
+        mockComplianceCheck.run_check = MagicMock(return_value=True)
+
+        return mockComplianceCheck
+
+    @pytest.fixture
+    def empty_compliance_suite(self):
+        return ComplianceSuite(
+            checks=[]
+        )
+
+    @pytest.fixture
+    def compliance_suite(self, mock_compliance_check):
+        return ComplianceSuite(
+            checks=[mock_compliance_check]
+        )
+
+    @pytest.fixture
+    def empty_compliance_results(self):
+        return []
+
+    @pytest.fixture
+    def compliance_results(self):
+        return [True]
+
+    def test_create_empty_compliance_suite(self, empty_compliance_suite):
+        assert len(empty_compliance_suite.checks) == 0
+
+    def test_create_compliance_suite(self, compliance_suite):
+        assert len(compliance_suite.checks) == 1
+
+    @pytest.mark.parametrize("suite,results", [
+        ("empty_compliance_suite", "empty_compliance_results"),
+        ("compliance_suite", "compliance_results")
+    ])
+    def test_run_compliance_suite(self, suite, results, request):
+        suite: ComplianceSuite = request.getfixturevalue(suite)
+        results: list = request.getfixturevalue(results)
+        assert suite.run("") == results
+
+        for check in suite.checks:
+            check.run_check.assert_called_once()
+
+
+class TestEndToEnd:
+    @pytest.mark.parametrize("card", [
+        ("""
+# Model Card for Sample Model
+
+Some random info...
+
+## Model Details
+
+### Model Description
+
+- **Developed by:** Nima Boscarino
+- **Model type:** Yada yada yada
+        """)
+    ])
+    def test_end_to_end_compliance_suite(self, card):
+        suite = ComplianceSuite(checks=[
+            ModelProviderIdentityCheck(),
+            IntendedPurposeCheck(),
+        ])
+
+        suite.run(card)