add support for additional backends (#33)

- add support for more backend (e37cee2506e5bdffbf68513d4e57d076c2c64c6f)
- remove uncessary warnings (0732974d483476a9eb15d1367abcb8f17069f57d)

app.py

@@ -1,5 +1,6 @@
 import gradio as gr
 
+import src.dependency  # noqa
 from src.assets import custom_css
 
 # from src.attention import create_attn_plots
@@ -14,10 +15,13 @@ from src.panel import (
     create_select_callback,
 )
 
-configs = load_hardware_configs("hardware.yml")
+configs = load_hardware_configs("hardware.yaml")
 
 
-demo = gr.Blocks(css=custom_css)
+demo = gr.Blocks(
+    css=custom_css,
+    theme=gr.themes.Default(primary_hue="indigo", secondary_hue="indigo"),
+)
 with demo:
     gr.HTML(LOGO, elem_classes="logo")
     gr.HTML(TITLE, elem_classes="title")
@@ -29,13 +33,13 @@ with demo:
                 if config.detail:
                     gr.Markdown(config.detail, elem_classes="descriptive-text")
 
-
-
                 # ####################### CONTROL PANEL #######################
                 (
                     filter_button,
-                    machine_textbox,
-                    subsets_values,
+                    machine_value,
+                    subsets_value,
+                    backends_value,
+                    hardware_type_value,
                     score_slider,
                     memory_slider,
                     backend_checkboxes,
@@ -43,18 +47,33 @@ with demo:
                     optimization_checkboxes,
                     quantization_checkboxes,
                     kernels_checkboxes,
-                ) = create_control_panel(machine=config.machine, subsets=config.subsets, hardware_provider=config.hardware_provider)
+                ) = create_control_panel(
+                    machine=config.machine,
+                    subsets=config.subsets,
+                    backends=config.backends,
+                    hardware_type=config.hardware_type,
+                    hardware_provider=config.hardware_provider,
+                )
                 ####################### HARDWARE SUBTABS #######################
                 with gr.Tabs(elem_classes="subtabs"):
-                    open_llm_perf_df = get_llm_perf_df(machine=config.machine, subsets=config.subsets)
+                    open_llm_perf_df = get_llm_perf_df(
+                        machine=config.machine,
+                        subsets=config.subsets,
+                        backends=config.backends,
+                        hardware_type=config.hardware_type,
+                    )
                     ####################### LEADERBOARD TAB #######################
                     with gr.TabItem("Leaderboard 🏅", id=0):
                         search_bar, columns_checkboxes, leaderboard_table = (
                             create_leaderboard_table(open_llm_perf_df)
                         )
-                    if config.hardware_provider != "intel": # TODO intel CPU does not measure the memory requirements correctly, so disable the graph feature until we fix the underlying issue
+                    if (
+                        config.hardware_provider != "intel"
+                    ):  # TODO intel CPU does not measure the memory requirements correctly, so disable the graph feature until we fix the underlying issue
                         with gr.TabItem("Find Your Best Model 🧭", id=1):
-                            lat_score_mem_plot = create_lat_score_mem_plot(open_llm_perf_df)
+                            lat_score_mem_plot = create_lat_score_mem_plot(
+                                open_llm_perf_df
+                            )
                     ###################### ATTENTIONS SPEEDUP TAB #######################
                     # with gr.TabItem("Attention 📈", id=2):
                     #     attn_prefill_plot, attn_decode_plot = create_attn_plots(
@@ -70,8 +89,10 @@ with demo:
                 create_control_callback(
                     filter_button,
                     # inputs
-                    machine_textbox,
-                    subsets_values,
+                    machine_value,
+                    subsets_value,
+                    backends_value,
+                    hardware_type_value,
                     score_slider,
                     memory_slider,
                     backend_checkboxes,
@@ -93,8 +114,10 @@ with demo:
 
                 create_select_callback(
                     # inputs
-                    machine_textbox,
-                    subsets_values,
+                    machine_value,
+                    subsets_value,
+                    backends_value,
+                    hardware_type_value,
                     # interactive
                     columns_checkboxes,
                     search_bar,

hardware.yml → hardware.yaml

@@ -1,7 +1,7 @@
 - machine: 1xA10
   description: A10-24GB-150W 🖥️
   hardware_provider: nvidia
-  hardware_type: gpu
+  hardware_type: cuda
   subsets:
     - unquantized
     - awq
@@ -13,7 +13,7 @@
 - machine: 1xA100
   description: A100-80GB-275W 🖥️
   hardware_provider: nvidia
-  hardware_type: gpu
+  hardware_type: cuda
   subsets:
     - unquantized
     - awq
@@ -25,7 +25,7 @@
 - machine: 1xT4
   description: T4-16GB-70W 🖥️
   hardware_provider: nvidia
-  hardware_type: gpu
+  hardware_type: cuda
   subsets:
     - unquantized
     - awq
@@ -43,4 +43,6 @@
   subsets:
     - unquantized
   backends:
-    - pytorch
+    - pytorch
+    - openvino
+    - onnxruntime

requirements.txt

@@ -1,5 +1,6 @@
 huggingface_hub
 transformers
-gradio
+gradio>=5.0.0
 plotly
-pandas
+pandas
+ruff

src/content.py

@@ -5,18 +5,18 @@ TITLE = """<h1 align="center" id="space-title">🤗 LLM-Perf Leaderboard 🏋️
 ABOUT = """
 ## 📝 About
 The 🤗 LLM-Perf Leaderboard 🏋️ is a laderboard at the intersection of quality and performance.
-Its aim is to benchmark the performance (latency, throughput, memory & energy) 
-of Large Language Models (LLMs) with different hardwares, backends and optimizations 
+Its aim is to benchmark the performance (latency, throughput, memory & energy)
+of Large Language Models (LLMs) with different hardwares, backends and optimizations
 using [Optimum-Benhcmark](https://github.com/huggingface/optimum-benchmark).
 
-Anyone from the community can request a new base model or hardware/backend/optimization 
+Anyone from the community can request a new base model or hardware/backend/optimization
 configuration for automated benchmarking:
 
-- Model evaluation requests should be made in the 
+- Model evaluation requests should be made in the
 [🤗 Open LLM Leaderboard 🏅](https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard) ;
 we scrape the [list of canonical base models](https://github.com/huggingface/optimum-benchmark/blob/main/llm_perf/utils.py) from there.
-- Hardware/Backend/Optimization configuration requests should be made in the 
-[🤗 LLM-Perf Leaderboard 🏋️](https://huggingface.co/spaces/optimum/llm-perf-leaderboard) or 
+- Hardware/Backend/Optimization configuration requests should be made in the
+[🤗 LLM-Perf Leaderboard 🏋️](https://huggingface.co/spaces/optimum/llm-perf-leaderboard) or
 [Optimum-Benhcmark](https://github.com/huggingface/optimum-benchmark) repository (where the code is hosted).
 
 ## ✍️ Details

src/dependency.py

@@ -0,0 +1,3 @@
+import os
+
+os.environ["TRANSFORMERS_NO_ADVISORY_WARNINGS"] = "1"

src/hardware.py

@@ -1,19 +1,19 @@
-from typing import Any, Dict, List
+from typing import Any, Dict, List, Optional
 
 import yaml
 
 
 class HardwareConfig:
     def __init__(self, data: Dict[str, Any]):
-        self.machine = data["machine"]
-        self.description = data["description"]
-        self.hardware_provider = data["hardware_provider"]
-        self.hardware_type = data["hardware_type"]
-        self.subsets = data["subsets"]
-        self.backends = data["backends"]
-        self.detail = data.get("detail", None)
+        self.machine: str = data["machine"]
+        self.description: str = data["description"]
+        self.hardware_provider: str = data["hardware_provider"]
+        self.hardware_type: str = data["hardware_type"]
+        self.subsets: List[str] = data["subsets"]
+        self.backends: List[str] = data["backends"]
+        self.detail: Optional[str] = data.get("detail", None)
 
-    def __repr__(self):
+    def __repr__(self) -> str:
         return (
             f"HardwareConfig(machine='{self.machine}', description='{self.description}', "
             f"hardware_provider={self.hardware_provider}, hardware_type={self.hardware_type}, subsets={self.subsets}, backends={self.backends})"

src/llm_perf.py

@@ -3,8 +3,6 @@ from typing import List
 
 import pandas as pd
 
-from src.hardware import HardwareConfig
-
 from .utils import process_kernels, process_quantizations
 
 DATASET_DIRECTORY = "dataset"
@@ -34,17 +32,31 @@ SORTING_COLUMNS = ["Open LLM Score (%)", "Decode (tokens/s)", "Prefill (s)"]
 SORTING_ASCENDING = [False, True, False]
 
 
-def get_raw_llm_perf_df(machine: str, subsets: List[str]):
+def get_raw_llm_perf_df(
+    machine: str, subsets: List[str], backends: List[str], hardware_type: str
+):
     dfs = []
     for subset in subsets:
-        try:
-            dfs.append(
-                pd.read_csv(
-                    f"hf://datasets/optimum-benchmark/llm-perf-leaderboard/perf-df-{subset}-{machine}.csv"
+        for backend in backends:
+            try:
+                dfs.append(
+                    pd.read_csv(
+                        f"hf://datasets/optimum-benchmark/llm-perf-leaderboard/perf-df-{backend}-{hardware_type}-{subset}-{machine}.csv"
+                    )
                 )
-            )
-        except Exception:
-            print(f"Subset {subset} for machine {machine} not found")
+            except Exception:
+                print("Dataset not found for:")
+                print(f"  • Backend: {backend}")
+                print(f"  • Subset: {subset}")
+                print(f"  • Machine: {machine}")
+                print(f"  • Hardware Type: {hardware_type}")
+                url = f"https://huggingface.co/datasets/optimum-benchmark/llm-perf-leaderboard/blob/main/perf-df-{backend}-{hardware_type}-{subset}-{machine}.csv"
+                print(f"  • URL: {url}")
+
+    if len(dfs) == 0:
+        raise ValueError(
+            f"No datasets found for machine {machine}, check your hardware.yml config file or your datatset on huggingface"
+        )
 
     perf_df = pd.concat(dfs)
     llm_df = pd.read_csv(
@@ -112,15 +124,22 @@ def processed_llm_perf_df(llm_perf_df):
     return llm_perf_df
 
 
-def get_llm_perf_df(machine: str, subsets: List[str]):
+def get_llm_perf_df(
+    machine: str, subsets: List[str], backends: List[str], hardware_type: str
+):
     if not os.path.exists(DATASET_DIRECTORY):
         os.makedirs(DATASET_DIRECTORY)
 
     if os.path.exists(f"{DATASET_DIRECTORY}/llm-perf-leaderboard-{machine}.csv"):
-        llm_perf_df = pd.read_csv(f"{DATASET_DIRECTORY}/llm-perf-leaderboard-{machine}.csv")
+        llm_perf_df = pd.read_csv(
+            f"{DATASET_DIRECTORY}/llm-perf-leaderboard-{machine}.csv"
+        )
     else:
-        llm_perf_df = get_raw_llm_perf_df(machine, subsets)
+        print(f"Dataset machine {machine} not found, downloading...")
+        llm_perf_df = get_raw_llm_perf_df(machine, subsets, backends, hardware_type)
         llm_perf_df = processed_llm_perf_df(llm_perf_df)
-        llm_perf_df.to_csv(f"{DATASET_DIRECTORY}/llm-perf-leaderboard-{machine}.csv", index=False)
+        llm_perf_df.to_csv(
+            f"{DATASET_DIRECTORY}/llm-perf-leaderboard-{machine}.csv", index=False
+        )
 
     return llm_perf_df

src/panel.py

@@ -10,17 +10,30 @@ from src.llm_perf import get_llm_perf_df
 from src.map import get_lat_score_mem_fig
 
 
-def create_control_panel(machine: str, subsets: List[str], hardware_provider: str):
+def create_control_panel(
+    machine: str,
+    subsets: List[str],
+    backends: List[str],
+    hardware_provider: str,
+    hardware_type: str,
+):
     # controls
-    machine_textbox = gr.Textbox(value=machine, visible=False)
-    subsets_values = gr.State(value=subsets)
-
+    machine_value = gr.State(value=machine)
+    subsets_value = gr.State(value=subsets)
+    backends_value = gr.State(value=backends)
+    hardware_type_value = gr.State(value=hardware_type)
 
     if hardware_provider == "nvidia":
         backends = ["pytorch"]
         attention_implementations = ["Eager", "SDPA", "FAv2"]
         quantizations = ["Unquantized", "BnB.4bit", "BnB.8bit", "AWQ.4bit", "GPTQ.4bit"]
-        kernels = ["No Kernel", "GPTQ.ExllamaV1", "GPTQ.ExllamaV2", "AWQ.GEMM", "AWQ.GEMV"]
+        kernels = [
+            "No Kernel",
+            "GPTQ.ExllamaV1",
+            "GPTQ.ExllamaV2",
+            "AWQ.GEMM",
+            "AWQ.GEMV",
+        ]
     elif hardware_provider == "intel":
         backends = ["pytorch", "onnxruntime", "openvino"]
         attention_implementations = ["Eager"]
@@ -29,7 +42,6 @@ def create_control_panel(machine: str, subsets: List[str], hardware_provider: st
     else:
         raise ValueError(f"Unknown hardware provider: {hardware_provider}")
 
-
     with gr.Accordion("Control Panel 🎛️", open=False, elem_id="control-panel"):
         with gr.Row():
             with gr.Column(scale=2, variant="panel"):
@@ -101,8 +113,10 @@ def create_control_panel(machine: str, subsets: List[str], hardware_provider: st
 
     return (
         filter_button,
-        machine_textbox,
-        subsets_values,
+        machine_value,
+        backends_value,
+        hardware_type_value,
+        subsets_value,
         score_slider,
         memory_slider,
         backend_checkboxes,
@@ -116,10 +130,12 @@ def create_control_panel(machine: str, subsets: List[str], hardware_provider: st
 def filter_rows_fn(
     machine,
     subsets,
+    backends,
+    hardware_type,
     # inputs
     score,
     memory,
-    backends,
+    backend_checkboxes,
     precisions,
     attentions,
     quantizations,
@@ -128,12 +144,14 @@ def filter_rows_fn(
     columns,
     search,
 ):
-    llm_perf_df = get_llm_perf_df(machine=machine, subsets=subsets)
+    llm_perf_df = get_llm_perf_df(
+        machine=machine, subsets=subsets, backends=backends, hardware_type=hardware_type
+    )
     # print(attentions)
     # print(llm_perf_df["Attention 👁️"].unique())
     filtered_llm_perf_df = llm_perf_df[
         llm_perf_df["Model 🤗"].str.contains(search, case=False)
-        & llm_perf_df["Backend 🏭"].isin(backends)
+        & llm_perf_df["Backend 🏭"].isin(backend_checkboxes)
         & llm_perf_df["Precision 📥"].isin(precisions)
         & llm_perf_df["Attention 👁️"].isin(attentions)
         & llm_perf_df["Quantization 🗜️"].isin(quantizations)
@@ -142,7 +160,7 @@ def filter_rows_fn(
         & (llm_perf_df["Memory (MB)"] <= memory)
     ]
     selected_filtered_llm_perf_df = select_columns_fn(
-        machine, subsets, columns, search, filtered_llm_perf_df
+        machine, subsets, backends, hardware_type, columns, search, filtered_llm_perf_df
     )
     selected_filtered_lat_score_mem_fig = get_lat_score_mem_fig(filtered_llm_perf_df)
     # filtered_bt_prefill_fig = get_bt_prefill_fig(filtered_df)
@@ -168,8 +186,10 @@ def create_control_callback(
     # button
     filter_button,
     # fixed
-    machine_textbox,
-    subsets_textbox,
+    machine_value,
+    subsets_value,
+    backends_value,
+    hardware_type_value,
     # inputs
     score_slider,
     memory_slider,
@@ -195,8 +215,10 @@ def create_control_callback(
         fn=filter_rows_fn,
         inputs=[
             # fixed
-            machine_textbox,
-            subsets_textbox,
+            machine_value,
+            subsets_value,
+            backends_value,
+            hardware_type_value,
             # inputs
             score_slider,
             memory_slider,
@@ -222,9 +244,16 @@ def create_control_callback(
     )
 
 
-def select_columns_fn(machine, subsets, columns, search, llm_perf_df=None):
+def select_columns_fn(
+    machine, subsets, backends, hardware_type, columns, search, llm_perf_df=None
+):
     if llm_perf_df is None:
-        llm_perf_df = get_llm_perf_df(machine=machine, subsets=subsets)
+        llm_perf_df = get_llm_perf_df(
+            machine=machine,
+            subsets=subsets,
+            backends=backends,
+            hardware_type=hardware_type,
+        )
 
     selected_leaderboard_df = get_leaderboard_df(llm_perf_df)
     selected_leaderboard_df = selected_leaderboard_df[
@@ -237,8 +266,10 @@ def select_columns_fn(machine, subsets, columns, search, llm_perf_df=None):
 
 def create_select_callback(
     # fixed
-    machine_textbox,
-    subsets_values,
+    machine_value,
+    subsets_value,
+    backends_value,
+    hardware_type_value,
     # interactive
     columns_checkboxes,
     search_bar,
@@ -247,11 +278,25 @@ def create_select_callback(
 ):
     columns_checkboxes.change(
         fn=select_columns_fn,
-        inputs=[machine_textbox, subsets_values, columns_checkboxes, search_bar],
+        inputs=[
+            machine_value,
+            subsets_value,
+            backends_value,
+            hardware_type_value,
+            columns_checkboxes,
+            search_bar,
+        ],
         outputs=[leaderboard_table],
     )
     search_bar.change(
         fn=select_columns_fn,
-        inputs=[machine_textbox, subsets_values, columns_checkboxes, search_bar],
+        inputs=[
+            machine_value,
+            subsets_value,
+            backends_value,
+            hardware_type_value,
+            columns_checkboxes,
+            search_bar,
+        ],
         outputs=[leaderboard_table],
     )