import gradio as gr import pandas as pd import json import os from constants import BANNER, INTRODUCTION_TEXT, CITATION_TEXT, METRICS_TAB_TEXT, DIR_OUTPUT_REQUESTS, LEADERBOARD_CSS from init import is_model_on_hub, upload_file, load_all_info_from_dataset_hub from utils_display import AutoEvalColumn, fields, make_clickable_model, make_clickable_paper, styled_error, styled_message, get_imagenet_columns import numpy as np from datetime import datetime, timezone def get_last_updated_date(file_path): """Get the last modification date of a file and format it nicely""" try: timestamp = os.path.getmtime(file_path) dt = datetime.fromtimestamp(timestamp) # Format as "Month DDth YYYY" day = dt.day if 4 <= day <= 20 or 24 <= day <= 30: suffix = "th" else: suffix = ["st", "nd", "rd"][day % 10 - 1] if day % 10 in [1, 2, 3] else "th" return dt.strftime(f"%b {day}{suffix} %Y") except OSError: return "Unknown" def load_models_list(json_path): """Load models list with paper and year information""" with open(json_path, 'r') as f: models_list = json.load(f) # Create a dictionary for quick lookup models_dict = {} for model in models_list: models_dict[model['path']] = { 'paper': model['paper'], 'year': model['year'], 'license': model['license'] } return models_dict def read_jsonl_to_dataframe(jsonl_path, models_dict): """Read JSONL file and convert to pandas DataFrame with proper unit conversions and model info""" data = [] with open(jsonl_path, 'r') as f: for line in f: if line.strip(): # Skip empty lines record = json.loads(line.strip()) # Convert units to match expected column names if 'parameters' in record: record['parameters_millions'] = record['parameters'] / 1_000_000 if 'flops' in record: record['flops_giga'] = record['flops'] / 1_000_000_000 if 'model_size' in record: record['model_size_mb'] = record['model_size'] / (1024 * 1024) # Add paper and year information if model exists in models list model_name = record.get('model', '') if model_name in models_dict: record['paper'] = models_dict[model_name]['paper'] record['year'] = str(models_dict[model_name]['year']) # Use license from models_dict if available, otherwise keep existing if 'license' not in record: record['license'] = models_dict[model_name]['license'] else: # Set default values for models not in the list record['paper'] = "N/A" record['year'] = "N/A" data.append(record) return pd.DataFrame(data) # Column names mapping for ImageNet-1k leaderboard column_names = { "model": "Model", "top1_accuracy": "Top-1 Accuracy ⬆️", "top5_accuracy": "Top-5 Accuracy ⬆️", "parameters_millions": "Parameters (M)", "flops_giga": "FLOPs (G)", "model_size_mb": "Model Size (MB)", "paper": "Paper", "year": "Year", "license": "License" } eval_queue_repo, requested_models, jsonl_results, _ = load_all_info_from_dataset_hub() if not jsonl_results.exists(): raise Exception(f"JSONL file {jsonl_results} does not exist locally") # Load models list with paper and year information models_dict = load_models_list("models_list.json") # Get jsonl with data and parse columns original_df = read_jsonl_to_dataframe(jsonl_results, models_dict) # Get last updated date from the jsonl file LAST_UPDATED = get_last_updated_date(jsonl_results) # Formats the columns def formatter(x): if type(x) is str: x = x elif x == -1 or pd.isna(x): x = "NA" else: x = round(x, 2) return x # Select only the columns we want to display in the final table display_columns = ['model', 'top1_accuracy', 'top5_accuracy', 'parameters_millions', 'flops_giga', 'model_size_mb', 'year','paper'] # Filter dataframe to only include display columns that exist available_columns = [col for col in display_columns if col in original_df.columns] filtered_df = original_df[available_columns].copy() # Format the columns for col in filtered_df.columns: if col == "model": filtered_df[col] = filtered_df[col].apply(lambda x: make_clickable_model(x)) elif col == "paper": filtered_df[col] = filtered_df[col].apply(lambda x: make_clickable_paper(x)) else: filtered_df[col] = filtered_df[col].apply(formatter) # For numerical values # Rename columns for display filtered_df.rename(columns=column_names, inplace=True) filtered_df.sort_values(by='Top-1 Accuracy ⬆️', ascending=False, inplace=True) # Update original_df to be the filtered version original_df = filtered_df # Get column definitions for ImageNet-1k imagenet_columns = get_imagenet_columns() COLS = [c.name for c in imagenet_columns] TYPES = [c.type for c in imagenet_columns] # ImageNet-1k specific functions (no multilingual functionality needed) def request_model(model_text, chb_imagenet): """Request evaluation of a model on ImageNet-1k dataset""" # Determine the selected checkboxes dataset_selection = [] if chb_imagenet: dataset_selection.append("ImageNet-1k validation set") if len(dataset_selection) == 0: return styled_error("You need to select at least one dataset") base_model_on_hub, error_msg = is_model_on_hub(model_text) if not base_model_on_hub: return styled_error(f"Base model '{model_text}' {error_msg}") # Construct the output dictionary current_time = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") required_datasets = ', '.join(dataset_selection) eval_entry = { "date": current_time, "model": model_text, "datasets_selected": required_datasets, "evaluation_type": "ImageNet-1k_classification" } # Prepare file path DIR_OUTPUT_REQUESTS.mkdir(parents=True, exist_ok=True) fn_datasets = '@ '.join(dataset_selection) filename = model_text.replace("/","@") + "@@" + fn_datasets if filename in requested_models: return styled_error(f"A request for this model '{model_text}' and dataset(s) was already made.") try: filename_ext = filename + ".txt" out_filepath = DIR_OUTPUT_REQUESTS / filename_ext # Write the results to a text file with open(out_filepath, "w") as f: f.write(json.dumps(eval_entry)) upload_file(filename, out_filepath) # Include file in the list of uploaded files requested_models.append(filename) # Remove the local file out_filepath.unlink() return styled_message("🤗 Your request has been submitted and will be evaluated soon!") except Exception as e: return styled_error(f"Error submitting request!") def filter_main_table(show_proprietary=True): filtered_df = original_df.copy() # Filter proprietary models if needed if not show_proprietary and "License" in filtered_df.columns: # Keep only models with "Open" license filtered_df = filtered_df[filtered_df["License"] == "Open"] return filtered_df with gr.Blocks(css=LEADERBOARD_CSS) as demo: gr.HTML(BANNER, elem_id="banner") gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text") with gr.Tabs(elem_classes="tab-buttons") as tabs: with gr.TabItem("🏅 Leaderboard", elem_id="imagenet-benchmark-tab-table", id=0): leaderboard_table = gr.components.Dataframe( value=original_df, datatype=TYPES, elem_id="leaderboard-table", interactive=False, visible=True, ) with gr.Row(): show_proprietary_checkbox = gr.Checkbox( label="Show proprietary models", value=True, elem_id="show-proprietary-checkbox" ) # Connect checkbox to the filtering function show_proprietary_checkbox.change( filter_main_table, inputs=[show_proprietary_checkbox], outputs=leaderboard_table ) with gr.TabItem("📈 Metrics", elem_id="imagenet-metrics-tab", id=1): gr.Markdown(METRICS_TAB_TEXT, elem_classes="markdown-text") with gr.TabItem("✉️✨ Request a model here!", elem_id="imagenet-request-tab", id=2): with gr.Column(): gr.Markdown("# ✉️✨ Request evaluation for a new model here!", elem_classes="markdown-text") with gr.Column(): gr.Markdown("Select a dataset:", elem_classes="markdown-text") with gr.Column(): model_name_textbox = gr.Textbox(label="Model name (user_name/model_name)") chb_imagenet = gr.Checkbox(label="ImageNet-1k validation set", value=True, interactive=True) with gr.Column(): mdw_submission_result = gr.Markdown() btn_submitt = gr.Button(value="🚀 Request Evaluation") btn_submitt.click(request_model, [model_name_textbox, chb_imagenet], mdw_submission_result) # add an about section with gr.TabItem("🤗 About", elem_id="imagenet-about-tab", id=3): gr.Markdown("## About", elem_classes="markdown-text") gr.Markdown(""" ### ImageNet-1k Leaderboard This leaderboard tracks the performance of computer vision models on the ImageNet-1k dataset, which is one of the most widely used benchmarks for image classification. #### Dataset Information - **Training images**: 1.2 million - **Validation images**: 50,000 - **Classes**: 1,000 object categories - **Image resolution**: Variable (typically 224×224 or 384×384) #### Hardware Configuration - **GPU**: NVIDIA L4 - All results are tested on the same hardware configuration to ensure fair comparison #### Evaluation Metrics - **Top-1 Accuracy**: Percentage of images where the top prediction is correct - **Top-5 Accuracy**: Percentage of images where the correct class is in top 5 predictions - **Parameters**: Number of trainable parameters in millions - **FLOPs**: Floating point operations in billions - **Model Size**: Size of the model file in MB #### Contributing To add your model to the leaderboard, use the "Request a model here!" tab. Your model will be evaluated on the ImageNet-1k validation set using NVIDIA L4 GPU and added to the leaderboard. """, elem_classes="markdown-text") gr.Markdown(f"Last updated on **{LAST_UPDATED}**", elem_classes="markdown-text") with gr.Row(): with gr.Accordion("📙 Citation", open=False): gr.Textbox( value=CITATION_TEXT, lines=7, label="Copy the BibTeX snippet to cite this source", elem_id="citation-button", show_copy_button=True, ) if __name__ == "__main__": demo.launch()