Add FP8 and fake_quant filter

Signed-off-by: jinsol-neubla <[email protected]>

app.py

@@ -3,7 +3,7 @@ import gradio as gr
 import pandas as pd
 from apscheduler.schedulers.background import BackgroundScheduler
 from huggingface_hub import snapshot_download
-from gradio_space_ci import enable_space_ci
+# from gradio_space_ci import enable_space_ci
 
 from src.display.about import (
     INTRODUCTION_TEXT,
@@ -25,6 +25,7 @@ from src.display.utils import (
     fields,
     WeightType,
     Precision,
+    Format
 )
 from src.envs import API, EVAL_RESULTS_PATH, RESULTS_REPO, REPO_ID, HF_TOKEN
 from src.populate import get_leaderboard_df
@@ -84,6 +85,7 @@ def update_table(
     activation_precision_query: str,
     size_query: list,
     hide_models: list,
+    format_query: list,
     query: str,
 ):
     filtered_df = filter_models(
@@ -93,6 +95,7 @@ def update_table(
         weight_precision_query=weight_precision_query,
         activation_precision_query=activation_precision_query,
         hide_models=hide_models,
+        format_query=format_query,
     )
     filtered_df = filter_queries(query, filtered_df)
     df = select_columns(filtered_df, columns)
@@ -153,6 +156,7 @@ def filter_models(
     weight_precision_query: list,
     activation_precision_query: list,
     hide_models: list,
+    format_query: list,
 ) -> pd.DataFrame:
     # Show all models
     if "Private or deleted" in hide_models:
@@ -175,6 +179,7 @@ def filter_models(
     filtered_df = filtered_df.loc[
         df[AutoEvalColumn.activation_precision.name].isin(activation_precision_query + ["None"])
     ]
+    filtered_df = filtered_df.loc[df[AutoEvalColumn.format.name].isin(format_query)]
 
     numeric_interval = pd.IntervalIndex(sorted([NUMERIC_INTERVALS[s] for s in size_query]))
     params_column = pd.to_numeric(df[AutoEvalColumn.params.name], errors="coerce")
@@ -191,6 +196,7 @@ leaderboard_df = filter_models(
     weight_precision_query=[i.value.name for i in Precision],
     activation_precision_query=[i.value.name for i in Precision],
     hide_models=["Private or deleted", "Contains a merge/moerge", "Flagged"],  # Deleted, merges, flagged, MoEs
+    format_query=[i.value.name for i in Format],
 )
 
 demo = gr.Blocks(css=custom_css)
@@ -227,7 +233,7 @@ with demo:
                     with gr.Row():
                         hide_models = gr.CheckboxGroup(
                             label="Hide models",
-                            choices=["Private or deleted", "Contains a merge/moerge", "Flagged", "MoE"],
+                            choices=["Private or deleted", "Contains a merge/moerge", "Flagged"], #, "MoE"],
                             value=["Private or deleted", "Contains a merge/moerge", "Flagged"],
                             interactive=True,
                         )
@@ -261,6 +267,13 @@ with demo:
                         interactive=True,
                         elem_id="filter-columns-size",
                     )
+                    filter_format = gr.CheckboxGroup(
+                        label="Format",
+                        choices=[i.value.name for i in Format],
+                        value=[i.value.name for i in Format],
+                        interactive=True,
+                        elem_id="filter-format",
+                    )
 
             leaderboard_table = gr.components.Dataframe(
                 value=leaderboard_df[
@@ -293,6 +306,7 @@ with demo:
                     filter_columns_activation_precision,
                     filter_columns_size,
                     hide_models,
+                    filter_format,
                     search_bar,
                 ],
                 leaderboard_table,
@@ -310,6 +324,7 @@ with demo:
                     filter_columns_activation_precision,
                     filter_columns_size,
                     hide_models,
+                    filter_format,
                     search_bar,
                 ],
                 leaderboard_table,
@@ -324,6 +339,7 @@ with demo:
                 filter_columns_activation_precision,
                 filter_columns_size,
                 hide_models,
+                filter_format,
             ]:
                 selector.change(
                     update_table,
@@ -335,6 +351,7 @@ with demo:
                         filter_columns_activation_precision,
                         filter_columns_size,
                         hide_models,
+                        filter_format,
                         search_bar,
                     ],
                     leaderboard_table,
@@ -374,4 +391,4 @@ scheduler = BackgroundScheduler()
 scheduler.add_job(restart_space, "interval", seconds=1800)  # restarted every 3h
 scheduler.start()
 
-demo.queue(default_concurrency_limit=40).launch()
+demo.queue(default_concurrency_limit=40).launch(share=True)

src/display/utils.py

@@ -66,6 +66,7 @@ auto_eval_column_dict.append(["flagged", ColumnContent, ColumnContent("Flagged",
 auto_eval_column_dict.append(["moe", ColumnContent, ColumnContent("MoE", "bool", False, hidden=True)])
 # Dummy column for the search bar (hidden by the custom CSS)
 auto_eval_column_dict.append(["dummy", ColumnContent, ColumnContent("model_name_for_query", "str", False, dummy=True)])
+auto_eval_column_dict.append(["format", ColumnContent, ColumnContent("Format", "str", False)])
 
 # We use make dataclass to dynamically fill the scores from Tasks
 AutoEvalColumn = make_dataclass("AutoEvalColumn", auto_eval_column_dict, frozen=True)
@@ -166,7 +167,9 @@ class Precision(Enum):
     float32 = ModelDetails("float32")
     float16 = ModelDetails("float16")
     bfloat16 = ModelDetails("bfloat16")
+    int8 = ModelDetails("int8")
     int4 = ModelDetails("int4")
+    float8 = ModelDetails("float8")
     Unknown = ModelDetails("?")
 
     def from_str(precision):
@@ -174,11 +177,25 @@ class Precision(Enum):
             return Precision.float16
         if precision in ["torch.bfloat16", "bfloat16"]:
             return Precision.bfloat16
+        if precision in ["int8"]:
+            return Precision.int8
         if precision in ["int4"]:
             return Precision.int4
+        if precision in ["float8", "fp8"]:
+            return Precision.float8
         if precision in ["torch.float32", "float32"]:
             return Precision.float32
         return Precision.Unknown
+    
+
+class Format(Enum):
+    FakeQuant = ModelDetails("FAKE_QUANT")
+    Unknown = ModelDetails("None")
+
+    def from_str(format):
+        if format in ["FAKE_QUANT"]:
+            return Format.FakeQuant
+        return Format.Unknown
 
 
 # Column selection

src/leaderboard/read_evals.py

@@ -36,6 +36,7 @@ class EvalResult:
     flagged: bool = False
     status: str = "FINISHED"
     tags: list = None
+    format: str = None
 
     @classmethod
     def init_from_json_file(self, json_filepath):
@@ -61,6 +62,8 @@ class EvalResult:
         weight_precision = Precision.from_str(config.get("weight_precision"))
         activation_precision = Precision.from_str(config.get("activation_precision"))
 
+        format = config.get("format", "None")
+
         # Get model and org
         org_and_model = config.get("model")
         org_and_model = org_and_model.split("/", 1)
@@ -78,25 +81,29 @@ class EvalResult:
         # Extract results available in this file (some results are split in several files)
         results = {}
         for task in Tasks:
-            task = task.value
-            # We skip old mmlu entries
-            # Some truthfulQA values are NaNs
-            if task.benchmark == "truthfulqa_mc2" and "truthfulqa_mc2|0" in data["results"]:
-                if math.isnan(float(data["results"]["truthfulqa_mc2|0"][task.metric])):
-                    results[task.benchmark] = 0.0
+            try:
+                task = task.value
+                # We skip old mmlu entries
+                # Some truthfulQA values are NaNs
+                if task.benchmark == "truthfulqa_mc2" and "truthfulqa_mc2|0" in data["results"]:
+                    if math.isnan(float(data["results"]["truthfulqa_mc2|0"][task.metric])):
+                        results[task.benchmark] = 0.0
+                        continue
+
+                # We average all scores of a given metric (mostly for mmlu)
+                if task.benchmark == "mmlu":
+                    accs = np.array([data["results"].get(task.benchmark, {}).get(task.metric, None)])
+                else:
+                    accs = np.array([v.get(task.metric, None) for k, v in data["results"].items() if task.benchmark in k])
+                if accs.size == 0 or any([acc is None for acc in accs]):
                     continue
 
-            # We average all scores of a given metric (mostly for mmlu)
-            if task.benchmark == "mmlu":
-                accs = np.array([data["results"].get(task.benchmark).get(task.metric, None)])
-            else:
-                accs = np.array([v.get(task.metric, None) for k, v in data["results"].items() if task.benchmark in k])
-            if accs.size == 0 or any([acc is None for acc in accs]):
+                mean_acc = np.mean(accs) * 100.0
+                results[task.benchmark] = mean_acc
+            except Exception as e:
+                print(e)
                 continue
 
-            mean_acc = np.mean(accs) * 100.0
-            results[task.benchmark] = mean_acc
-
         return self(
             eval_name=result_key,
             full_model=full_model,
@@ -112,6 +119,7 @@ class EvalResult:
             date=date,
             architecture=architecture,
             tags=tags,
+            format=format,
         )
 
     # def update_with_request_file(self, requests_path):
@@ -160,6 +168,7 @@ class EvalResult:
             AutoEvalColumn.merged.name: "merge" in self.tags if self.tags else False,
             AutoEvalColumn.moe.name: ("moe" in self.tags if self.tags else False) or "moe" in self.full_model.lower(),
             AutoEvalColumn.flagged.name: self.flagged,
+            AutoEvalColumn.format.name: self.format,
         }
 
         for task in Tasks: