app.py

import json

import datasets
import gradio as gr
import torch

from backend import (get_message_single, get_message_spam, send_single,
                     send_spam, tokenizer)
from defaults import (ADDRESS_BETTERTRANSFORMER, ADDRESS_VANILLA,
                      defaults_bt_single, defaults_bt_spam,
                      defaults_vanilla_single, defaults_vanilla_spam)


def dispatch_single(
    input_model_single, address_input_vanilla, address_input_bettertransformer
):
    result_vanilla = send_single(input_model_single, address_input_vanilla)
    result_bettertransformer = send_single(
        input_model_single, address_input_bettertransformer
    )

    return result_vanilla, result_bettertransformer


def dispatch_spam_artif(
    input_n_spam_artif,
    sequence_length,
    padding_ratio,
    address_input_vanilla,
    address_input_bettertransformer,
):
    sequence_length = int(sequence_length)
    input_n_spam_artif = int(input_n_spam_artif)

    inp_tokens = torch.randint(tokenizer.vocab_size - 1, (sequence_length,)) + 1

    n_pads = max(int(padding_ratio * len(inp_tokens)), 1)
    inp_tokens[-n_pads:] = 0

    inp_tokens[0] = 101
    inp_tokens[-n_pads - 1] = 102

    attention_mask = torch.zeros((sequence_length,), dtype=torch.int64)
    attention_mask[:-n_pads] = 1

    str_input = json.dumps(
        {
            "input_ids": inp_tokens.cpu().tolist(),
            "attention_mask": attention_mask.cpu().tolist(),
            "pre_tokenized": True,
        }
    )

    input_dataset = datasets.Dataset.from_dict(
        {"sentence": [str_input for _ in range(input_n_spam_artif)]}
    )

    result_vanilla = send_spam(input_dataset, address_input_vanilla)
    result_bettertransformer = send_spam(input_dataset, address_input_bettertransformer)

    return result_vanilla, result_bettertransformer


TTILE_IMAGE = """
<div
    style="
        display: block;
        margin-left: auto;
        margin-right: auto;
        width: 50%;
    "
>
<img src="https://huggingface.co/spaces/fxmarty/bettertransformer-demo/resolve/main/header_center.png"/>
</div>
"""

with gr.Blocks() as demo:
    gr.HTML(TTILE_IMAGE)
    gr.Markdown(
        "## Speed up inference and support more workload with PyTorch's BetterTransformer 🤗"
    )

    gr.Markdown(
    """
    **The two AWS instances powering this Space are offline (to save us the $$$). Feel free to reproduce using [this backend code](https://github.com/fxmarty/bettertransformer_demo). The example results are from an AWS EC2 g4dn.xlarge instance with a single NVIDIA T4 GPU.**
    """
    )

    gr.Markdown(
        """
    Let's try out [BetterTransformer](https://pytorch.org/blog/a-better-transformer-for-fast-transformer-encoder-inference/) + [TorchServe](https://pytorch.org/serve/)!

    BetterTransformer is a stable feature made available with [PyTorch 1.13](https://pytorch.org/blog/PyTorch-1.13-release/) allowing to use a fastpath execution for encoder attention blocks. Depending on your hardware, batch size, sequence length, padding ratio, it can bring large speedups at inference **at no cost in prediction quality**. As a one-liner, you can convert your 🤗 Transformers models to use BetterTransformer thanks to the integration in the [🤗 Optimum](https://github.com/huggingface/optimum) library:

    ```
    from optimum.bettertransformer import BetterTransformer

    better_model = BetterTransformer.transform(model)
    ```

    This Space is a demo of an **end-to-end** deployement of PyTorch eager-mode models, both with and without BetterTransformer. The goal is to see what are the benefits server-side and client-side of using BetterTransformer. The model used is [`distilbert-base-uncased-finetuned-sst-2-english`](https://huggingface.co/distilbert-base-uncased-finetuned-sst-2-english), and TorchServe is parametrized to use a maximum batch size of 8. **Beware:** you may be queued in case several persons use the Space at the same time.

    For more details on the TorchServe implementation and to reproduce, see [this reference code](https://github.com/fxmarty/bettertransformer_demo). For more details on BetterTransformer, check out the [blog post on PyTorch's Medium](https://medium.com/pytorch/bettertransformer-out-of-the-box-performance-for-huggingface-transformers-3fbe27d50ab2), and [the Optimum documentation](https://huggingface.co/docs/optimum/bettertransformer/overview)!"""
    )

    gr.Markdown("""## Single input scenario
    Note: BetterTransformer normally shines with batch size > 1 and some padding. So this is not the best case here. Check out the heavy workload case below as well.
    """)

    address_input_vanilla = gr.Textbox(
        max_lines=1, label="ip vanilla", value=ADDRESS_VANILLA, visible=False
    )

    address_input_bettertransformer = gr.Textbox(
        max_lines=1,
        label="ip bettertransformer",
        value=ADDRESS_BETTERTRANSFORMER,
        visible=False,
    )

    input_model_single = gr.Textbox(
        max_lines=1,
        label="Text",
        value="Expectations were low, enjoyment was high. Although the music was not top level, the story was well-paced.",
    )

    btn_single = gr.Button("Send single text request")
    with gr.Row():
        with gr.Column(scale=50):
            gr.Markdown("### Vanilla Transformers + TorchServe")
            output_single_vanilla = gr.Markdown(
                label="Output single vanilla",
                value=get_message_single(**defaults_vanilla_single),
            )
        with gr.Column(scale=50):
            gr.Markdown("### BetterTransformer + TorchServe")
            output_single_bt = gr.Markdown(
                label="Output single bt", value=get_message_single(**defaults_bt_single)
            )

    btn_single.click(
        fn=dispatch_single,
        inputs=[
            input_model_single,
            address_input_vanilla,
            address_input_bettertransformer,
        ],
        outputs=[output_single_vanilla, output_single_bt],
    )

    gr.Markdown(
        """
    **Beware that the end-to-end latency can be impacted by a different ping time between the two servers.**
    
    ## Heavy workload scenario
    """
    )

    input_n_spam_artif = gr.Number(
        label="Number of sequences to send",
        value=80,
    )
    sequence_length = gr.Number(
        label="Sequence length (in tokens)",
        value=128,
    )
    padding_ratio = gr.Number(
        label="Padding ratio (i.e. how much of the input is padding. In the real world when batch size > 1, the token sequence is padded with 0 to have all inputs with the same length.)",
        value=0.7,
    )
    btn_spam_artif = gr.Button("Spam text requests (using artificial data)")

    with gr.Row():
        with gr.Column(scale=50):
            gr.Markdown("### Vanilla Transformers + TorchServe")
            output_spam_vanilla_artif = gr.Markdown(
                label="Output spam vanilla",
                value=get_message_spam(**defaults_vanilla_spam),
            )
        with gr.Column(scale=50):
            gr.Markdown("### BetterTransformer + TorchServe")
            output_spam_bt_artif = gr.Markdown(
                label="Output spam bt", value=get_message_spam(**defaults_bt_spam)
            )

    btn_spam_artif.click(
        fn=dispatch_spam_artif,
        inputs=[
            input_n_spam_artif,
            sequence_length,
            padding_ratio,
            address_input_vanilla,
            address_input_bettertransformer,
        ],
        outputs=[output_spam_vanilla_artif, output_spam_bt_artif],
    )


demo.queue(concurrency_count=1)
demo.launch()