tests/test_intended_purpose_check.py

import pytest

import markdown
from bs4 import BeautifulSoup
from compliance_checks.intended_purpose import (
    IntendedPurposeCheck, IntendedPurposeResult,
)

empty_template = """\
## Uses

<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
[More Information Needed]

### Direct Use

[More Information Needed]

<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->

### Downstream Use [optional]

<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->

[More Information Needed]

### Out-of-Scope Use

<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
[More Information Needed]
"""
model_card_template = """\
## Uses

Some info...

### Direct Use

Some more info.

### Downstream Use [optional]

[More Information Needed]

### Out-of-Scope Use

Here is some info about out-of-scope uses...
"""
albert_base_v2 = """\
# ALBERT Base v2

## Intended uses & limitations
Here is some info about direct uses...
"""
distilbert_base_cased_distilled_squad = """\
# DistilBERT base cased distilled SQuAD

## Uses

This model can be used for question answering.
"""
distilroberta_base = """\
# Model Card for DistilRoBERTa base

# Uses

You can use the raw model for masked language modeling, but it's mostly intended to be fine-tuned on a downstream task.
"""

clip = """\
# Model Card: CLIP

## Model Use
Stuff.

### Intended Use
Stuff.

#### Primary intended uses
Stuff.

### Out-of-Scope Use Cases
Stuff.
"""

sentence_transformers = """\
# all-MiniLM-L6-v2

## Intended uses

Our model is intented to be used as a sentence and short paragraph encoder.
"""

bloom = """\
# BLOOM  

## Intended Use
This model is being created in order to enable public research on large language models (LLMs).
"""

bleed_over = """\
## Uses
"""

success_result = IntendedPurposeResult(
    status=True
)


@pytest.mark.parametrize("card", [
    model_card_template,
    albert_base_v2,
    distilbert_base_cased_distilled_squad,
    distilroberta_base,
    clip,
    sentence_transformers,
    bloom,
])
def test_run_checks(card):
    model_card_html = markdown.markdown(card)
    card_soup = BeautifulSoup(model_card_html, features="html.parser")

    results = IntendedPurposeCheck().run_check(card_soup)

    assert results == success_result


def test_fail_on_empty_template():
    model_card_html = markdown.markdown(empty_template)
    card_soup = BeautifulSoup(model_card_html, features="html.parser")
    results = IntendedPurposeCheck().run_check(card_soup)
    assert results == IntendedPurposeResult()


def test_fail_on_bleed_over():
    model_card_html = markdown.markdown(bleed_over)
    card_soup = BeautifulSoup(model_card_html, features="html.parser")
    results = IntendedPurposeCheck().run_check(card_soup)
    assert results == IntendedPurposeResult()