Upload folder using huggingface_hub

app/draw_diagram.py

@@ -0,0 +1,114 @@
+import streamlit as st
+import pandas as pd
+import numpy as np
+from streamlit_echarts import st_echarts
+# from streamlit_echarts import JsCode
+from streamlit_javascript import st_javascript
+# from PIL import Image 
+from app.show_examples import *
+
+links_dic = {}
+
+links_dic = {k.lower().replace('_', '-') : v for k, v in links_dic.items()}
+
+# huggingface_image = Image.open('style/huggingface.jpg')
+
+def nav_to(value):
+    try:
+        url = links_dic[str(value).lower()]
+        js = f'window.open("{url}", "_blank").then(r => window.parent.location.href);'
+        st_javascript(js)
+    except:
+        pass
+
+def draw(folder_name, category_name, dataset_name, metrics):
+    
+    folder = f"./results/{metrics}/"
+
+    display_names = {
+        'SU': 'Speech Understanding',
+        'ASU': 'Audio Scene Understanding',
+        'VU': 'Voice Understanding'
+    }
+    
+    data_path = f'{folder}/{category_name.lower()}.csv'
+    chart_data = pd.read_csv(data_path).round(2)
+
+    # if sorted == 'Ascending':
+    #     ascend = True 
+    # else:
+    #     ascend = False
+
+    new_dataset_name = dataset_name.replace('-', '_').lower()
+    chart_data = chart_data[['Model', new_dataset_name]]
+    
+    chart_data = chart_data.sort_values(by=[new_dataset_name], ascending=True).dropna(axis=0)
+
+    if len(chart_data) == 0:
+        return
+    
+    min_value = round(chart_data.iloc[:, 1::].min().min() - 0.1, 1) 
+    max_value = round(chart_data.iloc[:, 1::].max().max() + 0.1, 1)         
+
+    options = {
+        "title": {"text": f"{display_names[folder_name.upper()]}"},
+        "tooltip": {
+            "trigger": "axis",
+            "axisPointer": {"type": "cross", "label": {"backgroundColor": "#6a7985"}},
+            "triggerOn": 'mousemove',
+        },
+        "legend": {"data": ['Overall Accuracy']},
+        "toolbox": {"feature": {"saveAsImage": {}}},
+        "grid": {"left": "3%", "right": "4%", "bottom": "3%", "containLabel": True},
+        "xAxis": [
+            {
+                "type": "category",
+                "boundaryGap": False,
+                "triggerEvent": True,
+                "data": chart_data['Model'].tolist(),
+            }
+        ],
+        "yAxis": [{"type": "value", 
+                    "min": min_value,
+                    "max": max_value, 
+                    # "splitNumber": 10
+                    }],
+        "series": [{
+                "name": f"{dataset_name}",
+                "type": "line",
+                "data": chart_data[f'{new_dataset_name}'].tolist(),
+            }],
+    }
+    
+    events = {
+        "click": "function(params) { return params.value }"
+    }
+
+    value = st_echarts(options=options, events=events, height="500px")
+    
+    if value != None:
+        # print(value)
+        nav_to(value)
+
+    # if value != None:
+    #     highlight_table_line(value)
+
+    '''
+    Show table
+    '''
+    # st.divider()
+    with st.expander('TABLE'):
+    # chart_data['Link'] = chart_data['Model'].map(links_dic)
+        st.dataframe(chart_data,
+                    #  column_config = {
+                    #      "Link": st.column_config.LinkColumn(
+                    #          display_text= st.image(huggingface_image)
+                    #      ),
+                    #  }, 
+                        hide_index = True, 
+                        use_container_width=True)
+    '''
+    show samples
+    '''
+    show_examples(category_name, dataset_name, chart_data['Model'].tolist())
+    

app/pages.py

@@ -0,0 +1,303 @@
+import streamlit as st
+from app.draw_diagram import *
+
+def dashboard():
+
+    with st.container():
+        st.title("AudioBench")
+   
+        st.markdown("""
+            [gh]: https://github.com/AudioLLMs/AudioBench
+            [![GitHub watchers](https://img.shields.io/github/watchers/AudioLLMs/AudioBench?style=social)][gh]
+            [![GitHub Repo stars](https://img.shields.io/github/stars/AudioLLMs/AudioBench?style=social)][gh]
+            """)
+
+    audio_url = "https://arxiv.org/abs/2406.16020"
+
+    st.divider()
+    st.markdown("#### [AudioBench](%s)" % audio_url)
+    st.markdown("##### :dizzy: A comprehensive evaluation benchmark designed for general instruction-following audiolanguage models")
+    st.markdown('''
+
+                
+                ''')
+
+    with st.container():
+        left_co, center_co, right_co = st.columns([0.5,1, 0.5])
+        with center_co:
+            st.image("./style/audio_overview.png", 
+                     caption="Overview of the datasets in AudioBench.", 
+                     use_column_width = True)
+        
+        st.markdown('''
+
+                
+                ''')
+        
+        st.markdown("###### :dart: Our Benchmark includes: ")
+        cols = st.columns(10)
+        cols[1].metric(label="Tasks", value="8") #delta="Tasks", delta_color="off"
+        cols[2].metric(label="Datasets", value="26")
+        cols[3].metric(label="Test Models", value="5")
+
+        # st.markdown("###### :dart: Supported Models and Datasets: ")
+        
+        # sup = pd.DataFrame(
+        #         {"Dataset": "LibriSpeech-Clean", 
+        #          "Category": st.selectbox('category', ['Speech Understanding']),
+        #          "Task": st.selectbox('task', ['Automatic Speech Recognition']),
+        #          "Metrics": st.selectbox('metrics', ['WER']),
+        #          "Status":True}
+        # )
+        
+        # st.data_editor(sup, num_rows="dynamic")
+
+
+    st.divider()
+    with st.container():
+        st.markdown("##### Citations")
+
+        st.markdown('''
+                    :round_pushpin: AudioBench Paper \n
+                        @article{wang2024audiobench,
+                            title={AudioBench: A Universal Benchmark for Audio Large Language Models},
+                            author={Wang, Bin and Zou, Xunlong and Lin, Geyu and Sun, Shuo and Liu, Zhuohan and Zhang, Wenyu and Liu, Zhengyuan and Aw, AiTi and Chen, Nancy F},
+                            journal={arXiv preprint arXiv:2406.16020},
+                            year={2024}
+                            }
+                    ''')
+
+def asr():
+    st.title("Automatic Speech Recognition")
+    
+    filters_levelone = ['LibriSpeech-Test-Clean', 
+                        'LibriSpeech-Test-Other', 
+                        'Common-Voice-15-En-Test', 
+                        'Peoples-Speech-Test', 
+                        'GigaSpeech-Test', 
+                        'Earnings21-Test', 
+                        'Earnings22-Test', 
+                        'Tedlium3-Test', 
+                        'Tedlium3-Longform-Test', 
+                        'IMDA-Part1-ASR-Test', 
+                        'IMDA-Part2-ASR-Test', 
+                        'IMDA-Part3-ASR-Test', 
+                        'IMDA-Part4-ASR-Test', 
+                        'IMDA-Part5-ASR-Test', 
+                        'IMDA-Part6-ASR-Test']
+    
+    left, center, _, middle,right = st.columns([0.2, 0.2, 0.2, 0.2 ,0.2])
+    
+    with left:
+        filter_1 = st.selectbox('Select Dataset', filters_levelone)
+    
+    # with middle:
+    #     if filter_1 == filters_levelone[0]:
+    #         sort_leveltwo = ['LibriSpeech-Test-Clean', 'LibriSpeech-Test-Other', 'Common-Voice-15-En-Test', 'Peoples-Speech-Test',
+    #                         'GigaSpeech-Test', 'Tedlium3-Test','Tedlium3-Longform-Test', 'Earning-21-Test', 'Earning-22-Test']
+    #     elif filter_1 == filters_levelone[1]:
+    #         sort_leveltwo = ['CN-College-Listen-Test', 'SLUE-P2-SQA5-Test', 'DREAM-TTS-Test', 'Public-SG-SpeechQA-Test']
+        
+    #     elif filter_1 == filters_levelone[2]:
+    #         sort_leveltwo = ['OpenHermes-Audio-Test', 'ALPACA-Audio-Test']
+        
+    #     sort = st.selectbox("Sort Dataset", sort_leveltwo)
+    
+    # with right:
+    #     sorted = st.selectbox('by', ['Ascending', 'Descending'])
+
+    if filter_1:
+        draw('su', 'ASR', filter_1, 'wer')
+    else:
+        draw('su', 'ASR', 'LibriSpeech-Test-Clean', 'wer')
+
+
+    ## examples
+    
+
+def sqa():
+    st.title("Speech Question Answering")
+    
+    binary = ['CN-College-Listen-MCQ-Test', 'DREAM-TTS-MCQ-Test']
+
+    rest = ['SLUE-P2-SQA5-Test', 
+            'Public-SG-Speech-QA-Test', 
+            'Spoken-Squad-v1']
+
+    filters_levelone = binary + rest
+    
+    left, center, _, middle,right = st.columns([0.2, 0.2, 0.2, 0.2 ,0.2])
+    
+    with left:
+        filter_1 = st.selectbox('Select Dataset', filters_levelone)
+
+    if filter_1:
+        if filter_1 in binary:
+            draw('su', 'SQA', filter_1, 'llama3_70b_judge_binary')
+        else:
+            draw('su', 'SQA', filter_1, 'llama3_70b_judge')
+    else:
+        draw('su', 'SQA', 'CN-College-Listen-Test', 'llama3_70b_judge_binary')
+
+def si():
+    st.title("Speech Question Answering")
+    
+    filters_levelone = ['OpenHermes-Audio-Test', 
+                        'ALPACA-Audio-Test']
+    
+    left, center, _, middle,right = st.columns([0.2, 0.2, 0.2, 0.2 ,0.2])
+    
+    with left:
+        filter_1 = st.selectbox('Select Dataset', filters_levelone)
+
+    if filter_1:
+        draw('su', 'SI', filter_1, 'llama3_70b_judge')
+    else:
+        draw('su', 'SI', 'OpenHermes-Audio-Test', 'llama3_70b_judge')
+
+def ac():
+    st.title("Audio Captioning")
+
+    filters_levelone = ['WavCaps-Test', 
+                        'AudioCaps-Test']
+    filters_leveltwo = ['Llama3-70b-judge', 'Meteor']
+    
+    left, center, _, middle,right = st.columns([0.2, 0.2, 0.2, 0.2 ,0.2])
+    
+    with left:
+        filter_1 = st.selectbox('Select Dataset', filters_levelone)
+    with middle:
+        metric = st.selectbox('Select Metric', filters_leveltwo)
+    
+    # with middle:
+    #     if filter_1 == filters_levelone[0]:
+    #         sort_leveltwo = ['Clotho-AQA-Test', 'WavCaps-QA-Test', 'AudioCaps-QA-Test']
+    #     elif filter_1 == filters_levelone[1]:
+    #         sort_leveltwo = ['WavCaps-Test', 'AudioCaps-Test']
+        
+    #     sort = st.selectbox("Sort Dataset", sort_leveltwo)
+    
+    # with right:
+    #     sorted = st.selectbox('by', ['Ascending', 'Descending'])
+
+    if filter_1 or metric:
+        draw('asu', 'AC',filter_1, metric.lower().replace('-', '_'))
+    else:
+        draw('asu', 'AC', 'WavCaps-Test', 'llama3_70b_judge')
+
+def asqa():
+    st.title("Audio Scene Question Answering")
+
+    filters_levelone = ['Clotho-AQA-Test', 
+                        'WavCaps-QA-Test', 
+                        'AudioCaps-QA-Test']
+    
+    left, center, _, middle,right = st.columns([0.2, 0.2, 0.2, 0.2 ,0.2])
+    
+    with left:
+        filter_1 = st.selectbox('Select Dataset', filters_levelone)
+    
+    if filter_1:
+        draw('asu', 'AQA',filter_1, 'llama3_70b_judge')
+    else:
+        draw('asu', 'AQA', 'Clotho-AQA-Test', 'llama3_70b_judge')
+
+def er():
+    st.title("Emotion Recognition")
+
+    filters_levelone = ['IEMOCAP-Emotion-Test', 
+                        'MELD-Sentiment-Test', 
+                        'MELD-Emotion-Test']
+    sort_leveltwo = []
+    
+    left, center, _, middle,right = st.columns([0.2, 0.2, 0.2, 0.2 ,0.2])
+    
+    with left:
+        filter_1 = st.selectbox('Select Dataset', filters_levelone)
+    
+    # with middle:
+    #     if filter_1 == filters_levelone[0]:
+    #         sort_leveltwo = ['IEMOCAP-Emotion-Test', 'MELD-Sentiment-Test', 'MELD-Emotion-Test']
+        
+    #     elif filter_1 == filters_levelone[1]:
+    #         sort_leveltwo = ['VoxCeleb1-Accent-Test']
+        
+    #     elif filter_1 == filters_levelone[2]:
+    #         sort_leveltwo = ['VoxCeleb1-Gender-Test', 'IEMOCAP-Gender-Test']
+        
+    #     sort = st.selectbox("Sort Dataset", sort_leveltwo)
+    
+    # with right:
+    #     sorted = st.selectbox('by', ['Ascending', 'Descending'])
+
+    if filter_1:
+        draw('vu', 'ER', filter_1, 'llama3_70b_judge_binary')
+    else:
+        draw('vu', 'ER', 'IEMOCAP-Emotion-Test', 'llama3_70b_judge_binary')
+
+def ar():
+    st.title("Accent Recognition")
+
+    filters_levelone = ['VoxCeleb-Accent-Test']
+    
+    left, center, _, middle,right = st.columns([0.2, 0.2, 0.2, 0.2 ,0.2])
+    
+    with left:
+        filter_1 = st.selectbox('Select Dataset', filters_levelone)
+
+
+    if filter_1:
+        draw('vu', 'AR', filter_1, 'llama3_70b_judge')
+    else:
+        draw('vu', 'AR', 'VoxCeleb-Accent-Test', 'llama3_70b_judge')
+
+def gr():
+    st.title("Emotion Recognition")
+
+    filters_levelone = ['VoxCeleb-Gender-Test', 
+                        'IEMOCAP-Gender-Test']
+    
+    left, center, _, middle,right = st.columns([0.2, 0.2, 0.2, 0.2 ,0.2])
+    
+    with left:
+        filter_1 = st.selectbox('Select Dataset', filters_levelone)
+    
+    if filter_1:
+        draw('vu', 'GR', filter_1, 'llama3_70b_judge_binary')
+    else:
+        draw('vu', 'GR', 'VoxCeleb1-Gender-Test', 'llama3_70b_judge_binary')
+
+def spt():
+    st.title("Speech Translation")
+
+    filters_levelone = ['Covost2-EN-ID-test', 
+                        'Covost2-EN-ZH-test',
+                        'Covost2-EN-TA-test', 
+                        'Covost2-ID-EN-test', 
+                        'Covost2-ZH-EN-test', 
+                        'Covost2-TA-EN-test']
+    
+    left, center, _, middle,right = st.columns([0.2, 0.2, 0.2, 0.2 ,0.2])
+    
+    with left:
+        filter_1 = st.selectbox('Select Dataset', filters_levelone)
+    
+    if filter_1:
+        draw('su', 'ST', filter_1, 'bleu')
+    else:
+        draw('su', 'ST', 'Covost2-EN-ID-test', 'bleu')
+
+def cnasr():
+    st.title("Chinese Automatic Speech Recognition")
+
+    filters_levelone = ['Aishell-ASR-ZH-Test']
+    
+    left, center, _, middle,right = st.columns([0.2, 0.2, 0.2, 0.2 ,0.2])
+    
+    with left:
+        filter_1 = st.selectbox('Select Dataset', filters_levelone)
+    
+    if filter_1:
+        draw('su', 'CNASR', filter_1, 'wer')
+    else:
+        draw('su', 'CNASR', 'Aishell-ASR-ZH-Test', 'wer')

app/show_examples.py

@@ -0,0 +1,129 @@
+import streamlit as st
+import datasets
+import numpy as np
+
+
+def show_examples(category_name, dataset_name, model_lists):
+    st.divider()
+    sample_folder = f"./examples/{category_name}/{dataset_name}"
+    dataset = datasets.load_from_disk(sample_folder)
+
+    for index in range(len(dataset)):
+
+        with st.expander(f'EXAMPLE {index+1}'):
+            col1, col2 = st.columns([0.3, 0.7], vertical_alignment="center")
+
+            with col1:
+                st.audio(f'{sample_folder}/sample_{index}.wav', format="audio/wav")
+                      
+            with col2:
+                with st.container():
+                    custom_css = """
+                                <style>
+                                .my-container-question {
+                                background-color: #F5EEF8;
+                                padding: 10px;
+                                border-radius: 10px;
+                                height: auto;
+                                }
+                                </style>
+                                """
+                    st.markdown(custom_css, unsafe_allow_html=True)
+                    
+                    if dataset_name in ['CN-College-Listen-MCQ-Test', 'DREAM-TTS-MCQ-Test']:
+                        
+                        choices = dataset[index]['other_attributes']['choices'] 
+                        if isinstance(choices, str):
+                            choices_text = choices
+                        elif isinstance(choices, list):
+                            choices_text = ' '.join(i for i in choices)
+                        
+                        question_text = f"""<div class="my-container-question">
+                                         <p>QUESTION: {dataset[index]['instruction']['text']}</p>
+                                         <p>CHOICES: {choices_text}</p>
+                                         </div>
+                                         """
+                    else:
+                        question_text = f"""<div class="my-container-question">
+                                        <p>QUESTION: {dataset[index]['instruction']['text']}</p>
+                                        </div>"""
+                    
+                    
+                    st.markdown(question_text, unsafe_allow_html=True)
+                
+                with st.container():
+                    custom_css = """
+                                <style>
+                                .my-container-answer {
+                                background-color: #F9EBEA;
+                                padding: 10px;
+                                border-radius: 10px;
+                                height: auto;
+                                }
+                                </style>
+                                """
+                    st.markdown(custom_css, unsafe_allow_html=True)
+                    st.markdown(f"""<div class="my-container-answer">
+                                <p>CORRECT ANSWER: {dataset[index]['answer']['text']}</p>
+                                </div>""", unsafe_allow_html=True)
+
+            
+            st.divider()
+            with st.container():
+                custom_css = """
+                            <style>
+                            .my-container-table {
+                            background-color: #F2F3F4;
+                            padding: 10px;
+                            border-radius: 5px;
+                            # height: 50px;
+                            }
+                            </style>
+                            """
+                st.markdown(custom_css, unsafe_allow_html=True)
+
+                model_lists.sort()
+                
+                s = ''
+                if dataset_name in ['CN-College-Listen-MCQ-Test', 'DREAM-TTS-MCQ-Test']:
+                    for model in model_lists:
+                        try:
+                            s += f"""<tr>
+                                <td>{model}</td>
+                                <td><p>{dataset[index][model]['text']}</p> <p>{choices_text}</p></td>
+                                <td>{dataset[index][model]['model_prediction']}</td>
+                            </tr>"""
+                        except:
+                            print(f"{model} is not in {dataset_name}")
+                            continue
+                else:
+                    for model in model_lists:
+                        try:
+                            s += f"""<tr>
+                                <td>{model}</td>
+                                <td>{dataset[index][model]['text']}</td>
+                                <td>{dataset[index][model]['model_prediction']}</td>
+                            </tr>"""
+                        except:
+                            print(f"{model} is not in {dataset_name}")
+                            continue
+                
+                body_details = f"""<table style="width:100%">
+                <thead>
+                    <tr style="text-align: center;">
+                        <th style="width:20%">MODEL</th>
+                        <th style="width:40%">QUESTION</th>
+                        <th style="width:40%">MODEL PREDICTION</th>
+                    </tr>
+                {s}
+                </thead>
+                </table>"""
+                
+                st.markdown(f"""<div class="my-container-table">
+                                {body_details}
+                                </div>""", unsafe_allow_html=True)
+            
+                st.text("")
+
+    
+