Leaderboard updates, no more anonymous (tracking via huggingface

about.py

@@ -22,9 +22,51 @@ TODO
 We'd like to add some more existing models to the leaderboard. Some examples of models we'd like to add:
 - TODO
 
-**FAQs**
+### FAQs
 
 """
+# Note(Lood): Let's track these FAQs in the main Google Doc and have that remain the source of truth.
 FAQS = {
-    "Example FAQ with dropdown": """Full answer to this question""",
+    "Is there a fee to enter?": "No. Participation is free of charge.",
+    "Who can participate?": "Anyone. We encourage academic labs, individuals, and especially industry teams who use developability models in production.",
+    "Where can I find more information about the methods used to generate the data?": (
+        "Our [PROPHET-Ab preprint](https://www.biorxiv.org/content/10.1101/2025.05.01.651684v1) described in detail the methods used to generate the training dataset. "
+        "Note: These assays may differ from previously published methods, and these correlations between literature data and experimental data are also described in the preprint. "
+        "These same methods are used to generate the heldout test data."
+    ),
+    "How were the heldout sequences designed?": (
+        "We sampled 80 paired antibody sequences from [OAS](https://opig.stats.ox.ac.uk/webapps/oas/). We tried to represent the range of germline variants, sequence identities to germline, and CDR3 lengths. "
+        "The sequences in the dataset are quite diverse as measured by pairwise sequence identity."
+    ),
+    "Do I need to design new proteins?": (
+        "No. This is just a predictive competition, which will be judged according to the correlation between predictions and experimental values. "
+        "There may be a generative round in the future."
+    ),
+    "Can I participate anonymously?": (
+        "Yes! Please create an anonymous Hugging Face account so that we can uniquely associate submissions. "
+        "Note that top participants will be contacted to identify themselves at the end of the tournament."
+    ),
+    "How is intellectual property handled?": (
+        "Participants retain IP rights to the methods they use and develop during the tournament. Read more details in our terms here [link]."
+    ),
+    "Do I need to submit my code / methods in order to participate?": (
+        "No, there are no requirements to submit code / methods and submitted predictions remain private." 
+        "We also have an optional field for including a short model description. "
+        "Top performing participants will be requested to identify themselves at the end of the tournament. "
+        "There will be one prize for the best open-source model, which will require code / methods to be available."
+    ),
+    "How are winners determined?": (
+        "There will be 6 prizes (one for each of the assay properties plus an “open-source” prize). "
+        "For the property-specific prizes, winners will be determined by the submission with the highest Spearman rank correlation coefficient on the private holdout set. "
+        "For the “open-source prize”, this will be determined by the highest average Spearman across all properties. "
+        "We reserve the right to award the open-source prize to a predictor with competitive results for a subset of properties (e.g. a top polyreactivity model)."
+    ),
+    "How does the open-source prize work?": (
+        "Participants who open-source their code and methods will be eligible for the open-source prize (as well as the other prizes)."
+    ),
+    "What do I need to submit?": (
+        "There is a '✉️ Submit' tab on the Hugging Face competition page to upload predictions for datasets - for each dataset participants need to submit a CSV containing a column for each property they would like to predict (e.g. called “HIC”), "
+        "and a row with the sequence matching the sequence in the input file. These predictions are then evaluated in the backend using the Spearman rank correlation between predictions and experimental values, "
+        "and these metrics are then added to the leaderboard. Predictions remain private and are not seen by other contestants."
+    ),
 }

app.py

@@ -10,6 +10,7 @@ from constants import (
     ASSAY_EMOJIS,
     ASSAY_DESCRIPTION,
     EXAMPLE_FILE_DICT,
+    LEADERBOARD_DISPLAY_COLUMNS,
 )
 from about import ABOUT_TEXT, FAQS
 from submit import make_submission
@@ -20,36 +21,25 @@ def format_leaderboard_table(df_results: pd.DataFrame, assay: str | None = None)
     # Having a submission time column, and a user column where the username is clickable (this is a pro for usability but con for anonymity)
     # full_df.rename(columns={'submission_time': 'submission time', 'problem_type': 'problem type'}, inplace=True)
     # to_show['user'] = to_show['user'].apply(lambda x: make_user_clickable(x)).astype(str)
-    column_order = ["model", "property", "spearman", "spearman_cross_val"]
     df = df_results.query("assay.isin(@ASSAY_RENAME.keys())").copy()
     if assay is not None:
         df = df[df["assay"] == assay]
-    df = df[column_order]
+    df = df[LEADERBOARD_DISPLAY_COLUMNS]
     return df.sort_values(by="spearman", ascending=False)
 
 
-# Cache the results to avoid multiple downloads
-_cached_results = None
-
-
-def get_cached_results():
-    global _cached_results
-    if _cached_results is None:
-        _cached_results = fetch_hf_results()
-    return _cached_results
-
-
 def get_leaderboard_object(assay: str | None = None):
-    filter_columns = ["model"]
+    filter_columns = ["dataset"]
     if assay is None:
         filter_columns.append("property")
     # TODO how to sort filter columns alphabetically?
+    # Bug: Can't leave search_columns empty because then it says "Column None not found in headers"
     Leaderboard(
         # TODO(Lood) check that this actually refreshes using the function
-        value=format_leaderboard_table(df_results=get_cached_results(), assay=assay),
+        value=format_leaderboard_table(df_results=fetch_hf_results(), assay=assay),
         datatype=["str", "str", "str", "number"],
-        select_columns=["model", "property", "spearman", "spearman_cross_val"],
-        search_columns=["model"],
+        select_columns=["model", "property", "spearman", "dataset"],
+        search_columns=["model"],  # Note(Lood): Would be nice to make this clear it's searching on model name
         filter_columns=filter_columns,
         every=60,
         render=True,
@@ -57,6 +47,7 @@ def get_leaderboard_object(assay: str | None = None):
 
 
 with gr.Blocks() as demo:
+    # TODO: Add Ginkgo logo here on the top right
     gr.Markdown("""
         ## Welcome to the Ginkgo Antibody Developability Benchmark!
 
@@ -75,9 +66,11 @@ with gr.Blocks() as demo:
                 width="50vw",  # 50% of the "viewport width"
             )
             gr.Markdown(ABOUT_TEXT)
-            for question, answer in FAQS.items():
-                with gr.Accordion(question):
-                    gr.Markdown(answer)
+            for i, (question, answer) in enumerate(FAQS.items()):
+                # Would love to make questions bold but accordion doesn't support it
+                question = f"{i+1}. {question}"
+                with gr.Accordion(question, open=False):
+                    gr.Markdown(f"*{answer}*")  # Italics for answers
 
         # Procedurally make these 5 tabs
         for assay in ASSAY_LIST:
@@ -100,7 +93,7 @@ with gr.Blocks() as demo:
             # Antibody Developability Submission
             Upload a CSV to get a score!
 
-            Please use your Hugging Face account name to submit your model - we use this to track separate submissions, but only Hugging Face/Ginkgo will see these usernames (unless you choose to make them public).
+            Please use your Hugging Face account name to submit your model - we use this to track separate submissions, and if you would like to remain anonymous please set up an anonymous huggingface account.
             Your submission will be evaluated and added to the leaderboard.
             """
             )
@@ -114,7 +107,7 @@ with gr.Blocks() as demo:
                 with gr.Column():
                     username_input = gr.Textbox(
                         label="Username",
-                        placeholder="Enter your Hugging Face username or leave blank for anonymous submissions",
+                        placeholder="Enter your Hugging Face username",
                         info="This will be used to track your submissions, and to update your results if you submit again.",
                     )
                     model_name_input = gr.Textbox(
@@ -123,7 +116,7 @@ with gr.Blocks() as demo:
                         info="This will be displayed on the leaderboard.",
                     )
                     model_description_input = gr.Textbox(
-                        label="Model Description",
+                        label="Model Description (optional)",
                         placeholder="Brief description of your model and approach",
                         info="Describe your model, training data, or methodology.",
                         lines=3,
@@ -141,13 +134,6 @@ with gr.Blocks() as demo:
                     )
                     submission_file = gr.File(label="Submission CSV")
 
-            # If username is empty, set to anonymous submission
-            username_input.change(
-                fn=lambda x: x if x and x.strip() else None,
-                inputs=username_input,
-                outputs=user_state,
-            )
-
             def update_submission_type_and_file(submission_type):
                 """
                 Based on the submission type selected in the dropdown,
@@ -203,7 +189,7 @@ with gr.Blocks() as demo:
     gr.Markdown(
         """
         <div style="text-align: center; font-size: 14px; color: gray; margin-top: 2em;">
-        📬 For questions or feedback, contact <a href="mailto:datapoints@ginkgobioworks.com">datapoints@ginkgobioworks.com</a> or visit the Community tab at the top of this page.
+        📬 For questions or feedback, contact <a href="mailto:antibodycompetition@ginkgobioworks.com">antibodycompetition@ginkgobioworks.com</a> or visit the Community tab at the top of this page.
         </div>
         """,
         elem_id="contact-footer",

constants.py

@@ -59,4 +59,6 @@ ORGANIZATION = "ginkgo-datapoints"
 SUBMISSIONS_REPO = f"{ORGANIZATION}/abdev-bench-submissions"
 RESULTS_REPO = f"{ORGANIZATION}/abdev-bench-results"
 
-ANONYMOUS_SUBMISSION_USERNAME = "anonymoussubmissions"
+# Leaderboard dataframes
+LEADERBOARD_RESULTS_COLUMNS = ["model", "assay", "spearman", "dataset", "user"]  # The columns expected from the results dataset
+LEADERBOARD_DISPLAY_COLUMNS = ["model", "property", "spearman", "dataset", "user"]  # After changing assay to property (pretty formatting)

submit.py

@@ -3,48 +3,26 @@ import tempfile
 from typing import BinaryIO
 import json
 
+from click import pass_obj
 import gradio as gr
 from datetime import datetime
 import uuid
 
-from constants import API, SUBMISSIONS_REPO, ANONYMOUS_SUBMISSION_USERNAME
+from constants import API, SUBMISSIONS_REPO
 from validation import validate_csv_file, validate_username
 
 
-def make_submission(
-    submitted_file: BinaryIO,
+def upload_submission(
+    file_path: Path,
     user_state,
-    submission_type: str = "GDPa1",
-    model_name: str = "",
-    model_description: str = "",
+    submission_type: str,
+    model_name: str,
+    model_description: str,
 ):
-    user_state = user_state or ANONYMOUS_SUBMISSION_USERNAME
-    validate_username(user_state)
-
-    model_name = model_name.strip()
-    model_description = model_description.strip()
-
-    if not model_name:
-        raise gr.Error("Please provide a model name.")
-    if not model_description:
-        raise gr.Error("Please provide a model description.")
-    if submitted_file is None:
-        raise gr.Error("Please upload a CSV file before submitting.")
-
-    file_path = submitted_file.name
-
-    if not file_path:
-        raise gr.Error("Uploaded file object does not have a valid file path.")
-
-    path_obj = Path(file_path)
-
-    if path_obj.suffix.lower() != ".csv":
-        raise gr.Error("File must be a CSV file. Please upload a .csv file.")
-
     timestamp = datetime.utcnow().isoformat()
     submission_id = str(uuid.uuid4())
 
-    with path_obj.open("rb") as f_in:
+    with file_path.open("rb") as f_in:
         file_content = f_in.read().decode("utf-8")
 
         validate_csv_file(file_content, submission_type)
@@ -58,7 +36,7 @@ def make_submission(
             "submission_time": timestamp,
             "evaluated": False,
             "user": user_state,
-            "model": model_name,
+            "model_name": model_name,
             "model_description": model_description,
             "csv_content": file_content,
             "dataset": submission_type,
@@ -77,4 +55,42 @@ def make_submission(
         )
         Path(tmp_name).unlink()
 
+def make_submission(
+    submitted_file: BinaryIO,
+    user_state,
+    submission_type: str = "GDPa1",
+    model_name: str = "",
+    model_description: str = "",
+):
+    user_state = user_state
+    validate_username(user_state)
+
+    model_name = model_name.strip()
+    model_description = model_description.strip()
+
+    if not model_name:
+        raise gr.Error("Please provide a model name.")
+    if not model_description:
+        raise gr.Error("Please provide a model description.")
+    if submitted_file is None:
+        raise gr.Error("Please upload a CSV file before submitting.")
+
+    file_path = submitted_file.name
+
+    if not file_path:
+        raise gr.Error("Uploaded file object does not have a valid file path.")
+
+    path_obj = Path(file_path)
+
+    if path_obj.suffix.lower() != ".csv":
+        raise gr.Error("File must be a CSV file. Please upload a .csv file.")
+
+    upload_submission(
+        file_path=path_obj,
+        user_state=user_state,
+        submission_type=submission_type,
+        model_name=model_name,
+        model_description=model_description,
+    )
+
     return "✅ Your submission has been received! Sit tight and your scores will appear on the leaderboard shortly."

utils.py

@@ -7,7 +7,9 @@ import pandas as pd
 from datasets import load_dataset
 from huggingface_hub import hf_hub_download
 
-from constants import API, SUBMISSIONS_REPO, RESULTS_REPO, ASSAY_RENAME
+from constants import API, SUBMISSIONS_REPO, RESULTS_REPO, ASSAY_RENAME, LEADERBOARD_RESULTS_COLUMNS
+
+pd.set_option('display.max_columns', None)
 
 # def make_user_clickable(name):
 #     link =f'https://huggingface.co/{name}'
@@ -23,10 +25,12 @@ def show_output_box(message):
 
 
 def fetch_hf_results():
-    ds = load_dataset(
-        RESULTS_REPO, split="no_low_spearman", download_mode="force_redownload"
-    )
-    df = pd.DataFrame(ds).drop_duplicates(subset=["model", "assay"])
+    # Should cache by default if not using force_redownload
+    df = load_dataset(
+        RESULTS_REPO, data_files="auto_submissions/metrics_all.csv",
+    )["train"].to_pandas()
+    assert all(col in df.columns for col in LEADERBOARD_RESULTS_COLUMNS), f"Expected columns {LEADERBOARD_RESULTS_COLUMNS} not found in {df.columns}. Missing columns: {set(LEADERBOARD_COLUMNS) - set(df.columns)}"
+    df = df.drop_duplicates(subset=["model", "assay"])
     df["property"] = df["assay"].map(ASSAY_RENAME)
     print(df.head())
     return df