app.py

from functools import partial
import json

# from datasets import load_dataset
import gradio as gr
# from huggingface_hub import get_hf_file_metadata, HfApi, hf_hub_download, hf_hub_url
# from huggingface_hub.repocard import metadata_load
import pandas as pd
import numpy as np

DATASETS = {
    "samsum": "SAMSum",
    "cnn": "CNN/DailyMail",
    "xsum": "XSum",
    "billsum": "BillSum",
    "multinews": "Multi-News",
}

MODELS = [
    "PEGASUS", #0
    "PEGASUS-X", #1
    "MTL-ABS", #2
    "BART SDPT/DAPT/TAPT", #3
    "Prefix-tuning", #4
    "ExtraPhrase", #5
    "Primera", #6
    "Se3", #7
    "DADS", #8
    "LML-LRS", #9
    "PSP", #10
    "Athena", #11
    "SPEC", #12
    "Z-Code++", #13
    "DIONYSUS", #14
    "COMPO", #15
    "UNISUMM", #16
    "Centrum", #17
    "ParaSum", #18
    "EFLRAS", #19
]

REPOS_PAPERS = {
    "PEGASUS": "https://github.com/google-research/pegasus", #0
    "PEGASUS-X": "https://github.com/google-research/pegasus", #1
    "MTL-ABS": "https://github.com/YiSyuanChen/MTL-ABS", #2
    "BART SDPT/DAPT/TAPT": "https://github.com/TysonYu/AdaptSum", #3
    "Prefix-tuning": "https://github.com/XiangLi1999/PrefixTuning", #4
    "ExtraPhrase": "https://github.com/loem-ms/ExtraPhrase", #5
    "Primera": "https://github.com/allenai/PRIMER", #6
    "Se3": "https://ojs.aaai.org/index.php/AAAI/article/view/21357", #7
    "DADS": "https://aclanthology.org/2022.findings-naacl.53.pdf", #8
    "LML-LRS": "https://dl.acm.org/doi/pdf/10.1145/3477495.3531908", #9
    "PSP": "https://aclanthology.org/2022.coling-1.553.pdf", #10
    "Athena": "https://www.sciencedirect.com/science/article/pii/S0925231223004794?casa_token=ptLMl-LZLbQAAAAA:9Aq7HEUf6dRrIg5MTj4hZm2eaWJSeTDKmnXxS52fkZ131ejkYHdZgGimL0TFCFXy57qF1k9KTKE​", #11
    "SPEC": "https://github.com/YiSyuanChen/SPEC", #12
    "Z-Code++": "https://arxiv.org/pdf/2208.09770.pdf", #13
    "DIONYSUS": "https://arxiv.org/pdf/2212.10018.pdf", #14
    "COMPO": "https://github.com/ozyyshr/Compo", #15
    "UNISUMM": "https://github.com/microsoft/UniSumm", #16
    "Centrum": "https://github.com/ratishsp/centrum", #17
    "ParaSum": "https://link.springer.com/chapter/10.1007/978-3-031-40289-0_9", #18
    "EFLRAS": "https://github.com/NLPlab-skku/SummaryXAI-QA/tree/main/Low-Resource-Sum", #19
}

TAXONOMY = [
    "Pre-training", #0
    "Centroid-based pre-training", #1
    "Data augmentation", #2
    "Segmentation", #3
    "Meta-learning", #4
    "Meta-transfer", #5
    "Extractive summarization", #6
    "Prefix tuning", #7
]

MODEL_TO_TAXONOMY = [
    TAXONOMY[0],
    TAXONOMY[0],
    TAXONOMY[5],
    TAXONOMY[0],
    TAXONOMY[7],
    TAXONOMY[2],
    TAXONOMY[0],
    TAXONOMY[3],
    TAXONOMY[2],
    TAXONOMY[4],
    TAXONOMY[0],
    TAXONOMY[3],
    TAXONOMY[5],
    TAXONOMY[0],
    TAXONOMY[0],
    TAXONOMY[2],
    TAXONOMY[0],
    TAXONOMY[1],
    TAXONOMY[6],
    TAXONOMY[5],
]

model_tax = np.array([MODELS, MODEL_TO_TAXONOMY]).transpose()

SAMSUM_DATA = [
    [model_tax[14][0], "base", model_tax[14][1], 0, 0, 39.60, 15.40, 30.10],
    [model_tax[14][0], "large", model_tax[14][1], 0, 0, 41.30, 16.20, 30.90],
    [model_tax[3][0], "SDPT w/RecAdam", model_tax[3][1], 300, 0, 45.23, 19.43, 35.37],
    [model_tax[3][0], "DAPT", model_tax[3][1], 300, 0, 41.22, 17.88, 32.40],
    [model_tax[3][0], "TAPT w/RecAdam", model_tax[3][1], 300, 0, 41.34, 17.88, 32.31],
    [model_tax[13][0], "large", model_tax[13][1], 0, 0, 26.50, 7.90, 20.50],
    [model_tax[13][0], "large", model_tax[13][1], 10, 0, 40.27, 17.40, 33.70],
    [model_tax[13][0], "large", model_tax[13][1], 100, 0, 47.60, 22.30, 38.70],
    [model_tax[16][0], "", model_tax[16][1], 0, 0, 22.17, 6.88, 17.08],
    [model_tax[16][0], "", model_tax[16][1], 10, 0, 43.89, 18.53, 34.76],
    [model_tax[16][0], "", model_tax[16][1], 100, 0, 46.93, 20.65, 37.28],
    [model_tax[8][0], "", model_tax[8][1], 10, 0, 32.50, 12.00, 27.00],
    [model_tax[8][0], "", model_tax[8][1], 100, 0, 43.90, 19.70, 36.10],
    [model_tax[15][0], "base, self-training", model_tax[15][1], 147, 0, 45.42, 21.23, 41.42],
    [model_tax[15][0], "large, self-training", model_tax[15][1], 147, 0, 49.78, 24.65, 45.41],
    [model_tax[15][0], "base, joint-training", model_tax[15][1], 147, 0, 44.89, 20.64, 40.58],
    [model_tax[15][0], "large, joint-training", model_tax[15][1], 147, 0, 49.14, 23.45, 44.35],
    [model_tax[12][0], "", model_tax[12][1], 10, 0, 46.06, 20.90, 40.34],
    [model_tax[12][0], "", model_tax[12][1], 100, 0, 51.94, 24.75, 46.97],
]

CNN_DATA = [
    [model_tax[13][0], "large", model_tax[13][1], 0, 0, 40.00, 17.30, 25.30],
    [model_tax[13][0], "large", model_tax[13][1], 10, 0, 40.00, 17.30, 25.30],
    [model_tax[13][0], "large", model_tax[13][1], 100, 0, 41.10, 18.40, 27.50],
    [model_tax[0][0], "large", model_tax[0][1], 0, 0, 32.90, 13.28, 29.38],
    [model_tax[0][0], "large", model_tax[0][1], 10, 0, 37.25, 15.84, 33.49],
    [model_tax[0][0], "large", model_tax[0][1], 100, 0, 40.28, 18.21, 37.03],
    [model_tax[1][0], "large", model_tax[1][1], 0, 0, 30.22, 11.88, 28.31],
    [model_tax[1][0], "large", model_tax[1][1], 10, 0, 36.12, 13.70, 30.26],
    [model_tax[1][0], "large", model_tax[1][1], 100, 0, 38.40, 17.02, 36.75],
    [model_tax[10][0], "", model_tax[10][1], 300, 0, 38.31, 15.94, 25.41],
    [model_tax[5][0], "", model_tax[5][1], 1000, 0, 34.47, 12.91, 31.36],
    [model_tax[9][0], "", model_tax[9][1], 10, 0, 39.34, 16.53, 25.40],
    [model_tax[9][0], "", model_tax[9][1], 100, 0, 39.94, 16.96, 26.09],
    [model_tax[19][0], "", model_tax[19][1], 10, 0, 39.50, 16.80, 25.72],
    [model_tax[19][0], "", model_tax[19][1], 100, 0, 40.53, 17.61, 26.64],
    [model_tax[18][0], "", model_tax[18][1], 200, 0, 40.81, 17.78, 36.94],
]

BILLSUM_DATA = [
    [model_tax[0][0], "large", model_tax[0][0], 0, 0, 41.02, 17.44, 25.24],
    [model_tax[0][0], "large", model_tax[0][0], 10, 0, 40.48, 18.49, 27.27],
    [model_tax[0][0], "large", model_tax[0][0], 100, 0, 44.78, 26.40, 34.40],
    [model_tax[1][0], "large", model_tax[1][1], 0, 0, 41.32, 18.04, 25.11],
    [model_tax[1][0], "large", model_tax[1][1], 10, 0, 42.55, 18.97, 26.92],
    [model_tax[1][0], "large", model_tax[1][1], 100, 0, 46.48, 27.77, 36.53],
    [model_tax[7][0], "LED base(512) w/Se3", model_tax[7][1], 10, 0, 46.94, 23.04, 29.29],
    [model_tax[7][0], "LED base(512) w/Se3", model_tax[7][1], 100, 0, 50.4, 27.73, 33.74],
    [model_tax[11][0], "", model_tax[11][1], 10, 0, 47.57, 24.14, 30.35],
    [model_tax[11][0], "", model_tax[11][1], 100, 0, 51.59, 29.36, 35.04],
    [model_tax[9][0], "", model_tax[9][1], 10, 0, 46.64, 25.07, 30.90],
    [model_tax[9][0], "", model_tax[9][1], 100, 0, 48.18, 27.18, 33.28],
    [model_tax[2][0], "", model_tax[2][1], 10, 0, 41.22, 18.61, 26.33],
    [model_tax[2][0], "", model_tax[2][1], 100, 0, 45.29, 22.74, 29.56],
    [model_tax[19][0], "", model_tax[19][1], 10, 0, 46.64, 25.07, 30.90],
    [model_tax[19][0], "", model_tax[19][1], 100, 0, 48.18, 27.18, 33.28],
]

XSUM_DATA = [
    [model_tax[0][0], "large", model_tax[0][1], 0, 0, 19.27, 3.00, 12.72],
    [model_tax[0][0], "large", model_tax[0][1], 10, 0, 19.39, 3.45, 14.02],
    [model_tax[0][0], "large", model_tax[0][1], 100, 0, 39.07, 16.44, 31.27],
    [model_tax[10][0], "", model_tax[10][1], 300, 0, 32.86, 11.27, 25.64],
    [model_tax[16][0], "", model_tax[16][1], 0, 0, 20.72, 3.62, 16.56],
    [model_tax[16][0], "", model_tax[16][1], 10, 0, 26.10, 7.20, 19.92],
    [model_tax[16][0], "", model_tax[16][1], 100, 0, 33.33, 11.36, 25.85],
    [model_tax[9][0], "", model_tax[9][1], 10, 0, 32.35, 11.86, 25.33],
    [model_tax[9][0], "", model_tax[9][1], 100, 0, 35.54, 13.94, 27.79],
    [model_tax[19][0], "", model_tax[19][1], 10, 0, 32.65, 12.10, 25.82],
    [model_tax[19][0], "", model_tax[19][1], 100, 0, 36.51, 14.55, 29.01],
    [model_tax[12][0], "", model_tax[12][1], 10, 0, 32.74, 10.90, 24.86],
    [model_tax[12][0], "", model_tax[12][1], 100, 0, 35.69, 12.88, 27.25],
    [model_tax[18][0], "", model_tax[18][1], 1000, 0, 21.15, 3.08, 15.91],
    [model_tax[4][0], "", model_tax[4][1], 100, 0, 35.20, 13.30, 28.10],
]

MN_DATA = [
    [model_tax[0][0], "large", model_tax[0][1], 0, 0, 36.54, 10.52, 18.67],
    [model_tax[0][0], "large", model_tax[0][1], 10, 0, 39.79, 12.56, 20.06],
    [model_tax[0][0], "large", model_tax[0][1], 100, 0, 41.04, 13.88, 21.52],
    [model_tax[6][0], "", model_tax[6][1], 0, 0, 39.09, 13.91, 19.19],
    [model_tax[6][0], "", model_tax[6][1], 10, 0, 44.02, 15.54, 22.03],
    [model_tax[6][0], "", model_tax[6][1], 100, 0, 46.01, 16.76, 22.91],
    [model_tax[17][0], "", model_tax[17][1], 0, 0, 43.5, 15.7, 22.4],
    [model_tax[17][0], "", model_tax[17][1], 10, 0, 43.4, 16.6, 22.2],
    [model_tax[17][0], "", model_tax[17][1], 100, 0, 45.7, 16.8, 23.2],
    [model_tax[19][0], "", model_tax[19][1], 10, 0, 43.60, 14.85, 20.70],
    [model_tax[19][0], "", model_tax[19][1], 100, 0, 45.55, 16.01, 22.12],
    [model_tax[2][0], "", model_tax[2][1], 10, 0, 38.88, 12.78, 19.88],
    [model_tax[2][0], "", model_tax[2][1], 100, 0, 39.64, 13.64, 20.45],
]

COL_NAMES = [
    "Rank",
    "Model",
    "Additional info",
    "Taxonomy",
    "Training samples",
    "ROUGE",
    "ROUGE-1",
    "ROUGE-2",
    "ROUGE-L",
]

data = {
    "samsum": pd.DataFrame(SAMSUM_DATA),
    "cnn": pd.DataFrame(CNN_DATA),
    "billsum": pd.DataFrame(BILLSUM_DATA),
    "xsum": pd.DataFrame(XSUM_DATA),
    "multinews": pd.DataFrame(MN_DATA),
}

def make_clickable(text, url):
    return "<u>[{}]({})</u>".format(text, url)

for dataset in data:
    data[dataset].columns = COL_NAMES[1:]
    data[dataset]["ROUGE"] = np.around(np.mean(data[dataset][["ROUGE-1", "ROUGE-2", "ROUGE-L"]], axis=1), decimals=2)
    data[dataset].sort_values("ROUGE", ascending=False, inplace=True) # to default sort by ROUGE
    # Add Rank column
    data[dataset].insert(0, COL_NAMES[0], range(1, 1 + len(data[dataset])))
    # Add link to papers/repos
    data[dataset]["Model"] = data[dataset]["Model"].apply(lambda x: make_clickable(x, REPOS_PAPERS[x]))
    print(data[dataset]["Model"])
    # data[dataset].drop("ROUGE", axis=1, inplace=True)


NUM_DATASETS = len(set(DATASETS))
NUM_MODELS = len(set(MODELS))

block = gr.Blocks()
with block:
    gr.Markdown(f"""
    Low-Resource Summarization (LRS) Leaderboard. 🤗 Refer to the [Survey on LRS](the paper will be published soon) for details on metrics, taxonomy and models.

    - **Total Datasets**: {NUM_DATASETS}
    - **Total Models**: {NUM_MODELS}
    - **Metric**: ROUGE Score
    """)

    with gr.Tabs():
         for dataset in data:
              dataset_name = DATASETS[dataset]
              with gr.TabItem(dataset_name):
                with gr.Row():
                     gr.Markdown(f"""
                     **{dataset_name}** leaderboard
                     - **ROUGE** is the average of ROUGE-1, ROUGE-2 and ROUGE-L
                     - **RANK** is defined following ROUGE column values
                     """)
                with gr.Row():
                    data_classification = gr.components.Dataframe(
                        data[dataset],
                        datatype=["markdown", "markdown", "markdown", "number", "number", "number", "number", "number"],
                        type="pandas",
                    )

    # gr.Markdown(r"""

    # Made with ❤️ for NLP. If this work is useful to you, please consider citing:

    # citare il survey!!!

    # ```bibtex
    # @article{muennighoff2022mteb,
    #     doi = {10.48550/ARXIV.2210.07316},
    #     url = {https://arxiv.org/abs/2210.07316},
    #     author = {Muennighoff, Niklas and Tazi, Nouamane and Magne, Lo{\"\i}c and Reimers, Nils},
    #     title = {MTEB: Massive Text Embedding Benchmark},
    #     publisher = {arXiv},
    #     journal={arXiv preprint arXiv:2210.07316},  
    #     year = {2022}
    # }
    # ```
    # """)

block.queue(max_size=10)
block.launch(share=True)