updates

app.py

@@ -51,9 +51,8 @@ api = HfApi(token=H4_TOKEN)
 
 
 def restart_space():
-    api.restart_space(
-        repo_id="gsaivinay/open_llm_leaderboard", token=H4_TOKEN
-    )
+    api.restart_space(repo_id="gsaivinay/open_llm_leaderboard", token=H4_TOKEN)
+
 
 # Rate limit variables
 RATE_LIMIT_PERIOD = 7
@@ -98,7 +97,7 @@ else:
     eval_queue_private, eval_results_private = None, None
 
 original_df = get_leaderboard_df(eval_results, eval_results_private, COLS, BENCHMARK_COLS)
-models = original_df["model_name_for_query"].tolist() # needed for model backlinks in their to the leaderboard
+models = original_df["model_name_for_query"].tolist()  # needed for model backlinks in their to the leaderboard
 
 to_be_dumped = f"models = {repr(models)}\n"
 
@@ -130,7 +129,9 @@ def add_new_eval(
         error_msg = f"Organisation or user `{model.split('/')[0]}`"
         error_msg += f"already has {num_models_submitted_in_period} model requests submitted to the leaderboard "
         error_msg += f"in the last {RATE_LIMIT_PERIOD} days.\n"
-        error_msg += "Please wait a couple of days before resubmitting, so that everybody can enjoy using the leaderboard 🤗"
+        error_msg += (
+            "Please wait a couple of days before resubmitting, so that everybody can enjoy using the leaderboard 🤗"
+        )
         return styled_error(error_msg)
 
     if model_type is None or model_type == "":
@@ -213,9 +214,29 @@ def change_tab(query_param: str):
 
 
 # Searching and filtering
-def update_table(hidden_df: pd.DataFrame, current_columns_df: pd.DataFrame, columns: list, type_query: list, precision_query: str, size_query: list, show_deleted: bool, query: str):
+def update_table(
+    hidden_df: pd.DataFrame,
+    current_columns_df: pd.DataFrame,
+    columns: list,
+    type_query: list,
+    precision_query: str,
+    size_query: list,
+    show_deleted: bool,
+    query: str,
+):
     filtered_df = filter_models(hidden_df, type_query, size_query, precision_query, show_deleted)
-def update_table(hidden_df: pd.DataFrame, current_columns_df: pd.DataFrame, columns: list, type_query: list, precision_query: str, size_query: list, show_deleted: bool, query: str):
+
+
+def update_table(
+    hidden_df: pd.DataFrame,
+    current_columns_df: pd.DataFrame,
+    columns: list,
+    type_query: list,
+    precision_query: str,
+    size_query: list,
+    show_deleted: bool,
+    query: str,
+):
     filtered_df = filter_models(hidden_df, type_query, size_query, precision_query, show_deleted)
     if query != "":
         filtered_df = search_table(filtered_df, query)
@@ -223,9 +244,11 @@ def update_table(hidden_df: pd.DataFrame, current_columns_df: pd.DataFrame, colu
 
     return df
 
+
 def search_table(df: pd.DataFrame, query: str) -> pd.DataFrame:
     return df[(df[AutoEvalColumn.dummy.name].str.contains(query, case=False))]
 
+
 def select_columns(df: pd.DataFrame, columns: list) -> pd.DataFrame:
     always_here_cols = [
         AutoEvalColumn.model_type_symbol.name,
@@ -237,8 +260,9 @@ def select_columns(df: pd.DataFrame, columns: list) -> pd.DataFrame:
     ]
     return filtered_df
 
+
 NUMERIC_INTERVALS = {
-    "Unknown": pd.Interval(-1, 0, closed="right"), 
+    "Unknown": pd.Interval(-1, 0, closed="right"),
     "< 1.5B": pd.Interval(0, 1.5, closed="right"),
     "~3B": pd.Interval(1.5, 5, closed="right"),
     "~7B": pd.Interval(6, 11, closed="right"),
@@ -247,9 +271,9 @@ NUMERIC_INTERVALS = {
     "60B+": pd.Interval(55, 10000, closed="right"),
 }
 
+
 def filter_models(
     df: pd.DataFrame, type_query: list, size_query: list, precision_query: list, show_deleted: bool
-    df: pd.DataFrame, type_query: list, size_query: list, precision_query: list, show_deleted: bool
 ) -> pd.DataFrame:
     # Show all models
     if show_deleted:
@@ -545,7 +569,10 @@ with demo:
                                 max_rows=5,
                             )
             with gr.Row():
-                gr.Markdown("# ✉️✨ Submit your model [here!](https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard)", elem_classes="markdown-text")
+                gr.Markdown(
+                    "# ✉️✨ Submit your model [here!](https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard)",
+                    elem_classes="markdown-text",
+                )
 
     with gr.Row():
         with gr.Accordion("📙 Citation", open=False):

src/assets/css_html_js.py

@@ -33,7 +33,7 @@ custom_css = """
     background: none;
     border: none;
 }
- 
+
 #search-bar {
     padding: 0px;
 }
@@ -83,7 +83,7 @@ table th:first-child {
 #filter_type label > .wrap{
     width: 103px;
 }
-#filter_type label > .wrap .wrap-inner{  
+#filter_type label > .wrap .wrap-inner{
     padding: 2px;
 }
 #filter_type label > .wrap .wrap-inner input{

src/display_models/get_model_metadata.py

@@ -1,17 +1,15 @@
 import glob
 import json
 import os
-import re
 import pickle
+import re
 from typing import List
 
 import huggingface_hub
+from accelerate import init_empty_weights
 from huggingface_hub import HfApi
 from tqdm import tqdm
-from transformers import AutoModel, AutoConfig
-from accelerate import init_empty_weights
-from transformers import AutoModel, AutoConfig
-from accelerate import init_empty_weights
+from transformers import AutoConfig, AutoModel
 
 from src.display_models.model_metadata_flags import DO_NOT_SUBMIT_MODELS, FLAGGED_MODELS
 from src.display_models.model_metadata_type import MODEL_TYPE_METADATA, ModelType, model_type_from_str
@@ -25,7 +23,6 @@ def get_model_infos_from_hub(leaderboard_data: List[dict]):
     try:
         with open("model_info_cache.pkl", "rb") as f:
             model_info_cache = pickle.load(f)
-    except (EOFError, FileNotFoundError):
     except (EOFError, FileNotFoundError):
         model_info_cache = {}
     try:
@@ -67,7 +64,7 @@ def get_model_infos_from_hub(leaderboard_data: List[dict]):
         if model_name not in model_size_cache:
             model_size_cache[model_name] = get_model_size(model_name, model_info)
         model_data[AutoEvalColumn.params.name] = model_size_cache[model_name]
-    
+
     # save cache to disk in pickle format
     with open("model_info_cache.pkl", "wb") as f:
         pickle.dump(model_info_cache, f)
@@ -101,7 +98,7 @@ def get_model_size(model_name, model_info):
             with init_empty_weights():
                 model = AutoModel.from_config(config, trust_remote_code=False)
             return round(sum(p.numel() for p in model.parameters() if p.requires_grad) / 1e9, 3)
-        except (EnvironmentError, ValueError, KeyError): # model config not found, likely private  
+        except (EnvironmentError, ValueError, KeyError):  # model config not found, likely private
             try:
                 size_match = re.search(size_pattern, model_name.lower())
                 size = size_match.group(0)

src/display_models/read_results.py

@@ -107,7 +107,7 @@ def parse_eval_result(json_filepath: str) -> Tuple[str, list[dict]]:
                 revision=model_sha,
                 results={benchmark: mean_acc},
                 precision=precision,  # todo model_type=, weight_type=
-                date=config.get("submission_date")
+                date=config.get("submission_date"),
             )
         )
 

src/load_from_hub.py

@@ -1,10 +1,10 @@
 import json
 import os
+from collections import defaultdict
 
 import pandas as pd
 from huggingface_hub import Repository
 from transformers import AutoConfig
-from collections import defaultdict
 
 from src.assets.hardcoded_evals import baseline, gpt4_values, gpt35_values
 from src.display_models.get_model_metadata import apply_metadata
@@ -23,7 +23,8 @@ def get_all_requested_models(requested_models_dir: str) -> set[str]:
         current_depth = root.count(os.sep) - requested_models_dir.count(os.sep)
         if current_depth == depth:
             for file in files:
-                if not file.endswith(".json"): continue
+                if not file.endswith(".json"):
+                    continue
                 with open(os.path.join(root, file), "r") as f:
                     info = json.load(f)
                     file_names.append(f"{info['model']}_{info['revision']}_{info['precision']}")

src/rate_limiting.py

@@ -1,5 +1,4 @@
-
-from datetime import datetime, timezone, timedelta
+from datetime import datetime, timedelta, timezone
 
 
 def user_submission_permission(submission_name, users_to_submission_dates, rate_limit_period):
@@ -12,5 +11,3 @@ def user_submission_permission(submission_name, users_to_submission_dates, rate_
     submissions_after_timelimit = [d for d in submission_dates if d > time_limit]
 
     return len(submissions_after_timelimit)
-
-