init - init project

.gitignore

@@ -0,0 +1,44 @@
+*.pyc
+model_infos.json
+space
+.venv
+results
+mteb
+**/.DS_Store
+
+# FastAPI
+**/__pycache__/
+.pytest_cache/
+.coverage
+htmlcov/
+
+# Gradio
+flagged/
+gradio_cached_examples/
+tmp/
+logs/
+
+# IDE
+.idea/
+.vscode/
+
+# 环境相关
+.env
+.env.*
+.memo.md
+
+# ui pages
+app/ui/pages/conversational.py
+app/ui/pages/figures-and-tables.py
+app/ui/pages/healthcare.py
+app/ui/pages/law.py
+app/ui/pages/long-context.py
+app/ui/pages/multilingual.py
+app/ui/pages/text.py
+app/ui/pages/text-to-photo.py
+app/ui/pages/text-to-text.py
+app/ui/pages/tech.py
+app/ui/pages/document-screenshot.py
+app/ui/pages/finance.py
+app/ui/pages/code.py
+app/ui/pages/german.py

app.py

@@ -0,0 +1,36 @@
+import os
+
+import streamlit as st
+
+from st_pages import add_page_title, get_nav_from_toml
+
+from app.backend.constant import LEADERBOARD_MAP
+from app.backend.data_engine import DataEngine
+
+# init pages
+with open("app/ui/pages/data_page.py", "r", encoding="utf-8") as f:
+    data_page = f.read()
+for leaderboard, group_names in LEADERBOARD_MAP.items():
+
+    for group_name in group_names:
+        path = os.path.join("app/ui/pages", f"{group_name}.py")
+        with open(path, "w", encoding="utf-8") as f:
+            f.write(data_page.replace("$group_name$", group_name)
+                    )
+# init global data engine
+data_engine = DataEngine()
+data_engine.jsons_to_df()
+st.session_state["data_engine"] = data_engine
+
+st.set_page_config(layout="wide")
+
+# load page tree
+nav = get_nav_from_toml(
+    "app/ui/pages_sections.toml"
+)
+
+pg = st.navigation(nav)
+
+add_page_title(pg)
+
+pg.run()

app/backend/__init__.py

@@ -0,0 +1 @@
+

app/backend/constant.py

@@ -0,0 +1,83 @@
+from enum import Enum
+
+
+class Navigation(Enum):
+    TEXT_LEADERBOARD = "Text Leaderboard"
+    MULTIMODAL_LEADERBOARD = "Multimodal Leaderboard"
+
+
+class TaskCategory(Enum):
+    LAW = "Law"
+    CODE = "Code"
+    CONVERSATIONAL = "Conversational"
+    TECH = "Tech"
+    LONG_CONTEXT = "Long-context"
+    MULTILINGUAL = "Multilingual"
+
+
+class ModelProvider(Enum):
+    OPENAI = "OpenAI"
+    VOYAGEAI = "VoyageAI"
+    COHERE = "Cohere"
+    OTHERS = "Others"
+
+
+class EvaluationMetric(Enum):
+    NDCG_1 = "NDCG@1"
+    NDCG_3 = "NDCG@3"
+    NDCG_5 = "NDCG@5"
+    NDCG_10 = "NDCG@10"
+    NDCG_20 = "NDCG@20"
+    NDCG_50 = "NDCG@50"
+    NDCG_100 = "NDCG@100"
+    RECALL_1 = "RECALL@1"
+    RECALL_3 = "RECALL@3"
+    RECALL_5 = "RECALL@5"
+    RECALL_10 = "RECALL@10"
+    RECALL_20 = "RECALL@20"
+    RECALL_50 = "RECALL@50"
+    RECALL_100 = "RECALL@100"
+    PRECISION_1 = "PRECISION@1"
+    PRECISION_3 = "PRECISION@3"
+    PRECISION_5 = "PRECISION@5"
+    PRECISION_10 = "PRECISION@10"
+    PRECISION_20 = "PRECISION@20"
+    PRECISION_50 = "PRECISION@50"
+    PRECISION_100 = "PRECISION@100"
+
+
+class EmbdDtype(Enum):
+    ALL = "all"
+    FLOAT_32 = "float32"
+    INT_8 = "int8"
+    BINARY = "binary"
+
+
+class EmbdDim(Enum):
+    OP1 = "<=1k"
+    OP2 = "1k-2k"
+    OP3 = "2k-5k"
+    OP4 = ">=5k"
+
+
+class Similarity(Enum):
+    ALL = "all"
+    COSINE = "cosine"
+    DOT = "dot"
+    EUCLIDEAN = "euclidean"
+
+
+LEADERBOARD_MAP = {
+    "Text": [
+        "text",
+        "law",
+        "multilingual",
+        "german",
+        "code",
+        "tech",
+
+    ],
+    "Multimodal": [
+
+    ]
+}

app/backend/data_engine.py

@@ -0,0 +1,117 @@
+"""
+Data service provider
+"""
+import json
+from typing import List
+
+import pandas as pd
+
+from utils.cache_decorator import cache_df_with_custom_key, cache_dict_with_custom_key
+from utils.http_utils import get
+
+COLUMNS = ['model_name',
+           'embd_dtype', 'embd_dim', 'num_params', 'max_tokens', 'similarity',
+           'query_instruct', 'corpus_instruct',
+           'ndcg_at_10',
+           ]
+COLUMNS_TYPES = ["markdown",
+                 'str', 'str', 'number', 'number', 'str',
+                 'str', 'str',
+                 'number',
+                 ]
+
+GIT_URL = "https://raw.githubusercontent.com/embedding-benchmark/ebr/refs/heads/main/results/"
+DATASET_URL = f"{GIT_URL}datasets.json"
+MODEL_URL = f"{GIT_URL}models.json"
+RESULT_URL = f"{GIT_URL}results.json"
+
+
+class DataEngine:
+
+    def __init__(self):
+        self.df = self.init_dataframe()
+
+    @property
+    @cache_dict_with_custom_key("models")
+    def models(self):
+        """
+        Get models data
+        """
+        res = get(MODEL_URL)
+        if res.status_code == 200:
+            return res.json()
+        return {}
+
+    @property
+    @cache_dict_with_custom_key("datasets")
+    def datasets(self):
+        """
+        Get tasks data
+        """
+        res = get(DATASET_URL)
+        if res.status_code == 200:
+            return res.json()
+        return {}
+
+    @property
+    @cache_dict_with_custom_key("results")
+    def results(self):
+        """
+        Get results data
+        """
+        res = get(RESULT_URL)
+        if res.status_code == 200:
+            return res.json()
+        return {}
+
+    def init_dataframe(self):
+        """
+        Initialize DataFrame
+        """
+        d = {"hello": [123], "world": [456]}
+        return pd.DataFrame(d)
+
+    @cache_df_with_custom_key("json_result")
+    def jsons_to_df(self):
+
+        results_list = self.results
+        df_results_list = []
+        for result_dict in results_list:
+            dataset_name = result_dict["dataset_name"]
+            df_result_row = pd.DataFrame(result_dict["results"])
+            df_result_row["dataset_name"] = dataset_name
+            df_results_list.append(df_result_row)
+        df_result = pd.concat(df_results_list)
+
+        df_datasets_list = []
+        for item in self.datasets:
+            dataset_names = item["datasets"]
+            df_dataset_row = pd.DataFrame(
+                {
+                    "group_name": [item["name"] for _ in range(len(dataset_names))],
+                    "dataset_name": dataset_names,
+                    "leaderboard": [item["leaderboard"] for _ in range(len(dataset_names))]
+                }
+            )
+            df_datasets_list.append(df_dataset_row)
+        df_dataset = pd.concat(df_datasets_list).drop_duplicates()
+
+        models_list = self.models
+
+        df_model = pd.DataFrame(models_list)
+
+        df = pd.merge(df_result, df_dataset, on=["dataset_name"], how="inner")
+        df = df.groupby(["model_name", "group_name"], as_index=False)["ndcg_at_10"].mean()
+        df = pd.merge(df, df_model, on=["model_name"], how="inner")
+
+        if df.empty:
+            return pd.DataFrame(columns=COLUMNS + ["group_name", "reference"])
+        return df[COLUMNS + ["group_name", "reference"]]
+
+    def filter_df(self, group_name: str):
+        """
+        filter_by_providers
+        """
+        df = self.jsons_to_df()
+
+        return df[df["group_name"] == group_name][COLUMNS][:]

app/ui/pages/data_page.py

@@ -0,0 +1,170 @@
+from st_aggrid import AgGrid, JsCode, ColumnsAutoSizeMode
+
+import streamlit as st
+
+HEADER_STYLE = {'fontSize': '18px'}
+CELL_STYLE = {'fontSize': '18px'}
+
+# Add theme color and grid styles
+st.markdown("""
+    <style>
+        :root {
+            --theme-color: rgb(129, 150, 64);
+            --theme-color-light: rgba(129, 150, 64, 0.2);
+        }
+        
+        /* AG Grid specific overrides */
+        .ag-theme-alpine {
+            --ag-selected-row-background-color: var(--theme-color-light) !important;
+            --ag-row-hover-color: var(--theme-color-light) !important;
+            --ag-selected-tab-color: var(--theme-color) !important;
+            --ag-range-selection-border-color: var(--theme-color) !important;
+            --ag-range-selection-background-color: var(--theme-color-light) !important;
+        }
+        
+        .ag-row-hover {
+            background-color: var(--theme-color-light) !important;
+        }
+        
+        .ag-row-selected {
+            background-color: var(--theme-color-light) !important;
+        }
+        
+        .ag-row-focus {
+            background-color: var(--theme-color-light) !important;
+        }
+        
+        .ag-cell-focus {
+            border-color: var(--theme-color) !important;
+        }
+        
+        /* Keep existing styles */
+        .center-text {
+            text-align: center;
+            color: var(--theme-color);
+        }
+        .center-image {
+            display: block;
+            margin-left: auto;
+            margin-right: auto;
+        }
+        h2 {
+            color: var(--theme-color) !important;
+        }
+        .ag-header-cell {
+            background-color: var(--theme-color) !important;
+            color: white !important;
+        }
+        a {
+            color: var(--theme-color) !important;
+        }
+        a:hover {
+            color: rgba(129, 150, 64, 0.8) !important;
+        }
+    </style>
+""", unsafe_allow_html=True)
+
+#  logo
+# st.markdown('<img src="https://www.voyageai.com/logo.svg" class="center-image" width="200">', unsafe_allow_html=True)
+
+# title
+st.markdown('<h2 class="center-text">Embedding Benchmark For Retrieval</h2>', unsafe_allow_html=True)
+
+group_name = "$group_name$"
+
+data_engine = st.session_state["data_engine"]
+
+df = data_engine.jsons_to_df()[:]
+
+df = df[df["group_name"] == group_name].sort_values(by="ndcg_at_10", ascending=False)
+
+# setting column config
+grid_options = {
+    'columnDefs': [
+        {
+            'headerName': 'Model Name',
+            'field': 'model_name',
+            'pinned': 'left',
+            'sortable': False,
+            'headerStyle': HEADER_STYLE,
+            'cellStyle': CELL_STYLE,
+            'cellRenderer': JsCode("""class CustomHTML {
+                init(params) {
+                    const link = params.data.reference;
+                    this.eGui = document.createElement('div');
+                    this.eGui.innerHTML = link ? 
+                        `<a href="${link}" target="_blank">${params.value}</a>` : 
+                        params.value;
+                }
+
+                getGui() {
+                    return this.eGui;
+                }
+            }"""),
+        },
+        {
+            'headerName': 'NDCG@10',
+            'field': 'ndcg_at_10',
+            'headerStyle': HEADER_STYLE,
+            'cellStyle': CELL_STYLE,
+        },
+        {
+            'headerName': 'Data Type',
+            'field': 'embd_dtype',
+            'headerStyle': HEADER_STYLE,
+            'cellStyle': CELL_STYLE,
+        },
+        {
+            'headerName': 'Embd Dim',
+            'field': 'embd_dim',
+            'headerStyle': HEADER_STYLE,
+            'cellStyle': CELL_STYLE,
+        },
+        {
+            'headerName': 'Model Size (# of Parameters)',
+            'field': 'num_params',
+            'cellDataType': 'number',
+            'headerStyle': HEADER_STYLE,
+            'cellStyle': CELL_STYLE,
+        },
+        {
+            'headerName': 'Context Length',
+            'field': 'max_tokens',
+            'headerStyle': HEADER_STYLE,
+            'cellStyle': CELL_STYLE,
+        },
+        {
+            'headerName': 'Query Instruction',
+            'field': 'query_instruct',
+            'headerStyle': HEADER_STYLE,
+            'cellStyle': CELL_STYLE,
+            'suppressSizeToFit': True,
+
+        },
+        {
+            'headerName': 'Corpus Instruction',
+            'field': 'corpus_instruct',
+            'headerStyle': HEADER_STYLE,
+            'cellStyle': CELL_STYLE,
+            'suppressSizeToFit': True,
+
+        },
+    ],
+    'defaultColDef': {
+        'filter': True,
+        'sortable': True,
+        'resizable': True
+    },
+    'autoSizeStrategy': {
+        'type': 'fitCellContents'
+    }
+}
+
+ag = AgGrid(
+    df,
+    enable_enterprise_modules=False,
+    gridOptions=grid_options,
+    allow_unsafe_jscode=True,
+    columns_auto_size_mode=ColumnsAutoSizeMode.FIT_CONTENTS,
+    theme="streamlit",
+)

app/ui/pages_sections.toml

@@ -0,0 +1,38 @@
+[[pages]]
+name = "Text Leaderboard"
+icon = "📚"
+is_section = true
+
+[[pages]]
+path = "app/ui/pages/text.py"
+name = "Overall"
+icon = "🏆"
+
+[[pages]]
+path = "app/ui/pages/law.py"
+name = "Law"
+icon = "⚖️"
+
+[[pages]]
+path = "app/ui/pages/multilingual.py"
+name = "Multilingual"
+icon = "🌎"
+
+[[pages]]
+path = "app/ui/pages/german.py"
+name = "German"
+icon = "🇩🇪"
+
+[[pages]]
+path = "app/ui/pages/code.py"
+name = "Code"
+icon = "💻"
+
+[[pages]]
+path = "app/ui/pages/tech.py"
+name = "Tech"
+icon = "🛠️"
+
+
+
+

requirements.txt

@@ -0,0 +1,3 @@
+streamlit==1.41.1
+streamlit-aggrid==1.0.5
+st-pages==1.0.1

utils/cache_decorator.py

@@ -0,0 +1,54 @@
+import time
+from functools import wraps
+import pandas as pd
+
+CACHE = {}
+TTL = 3600
+
+
+def cache_df_with_custom_key(cache_key: str):
+    def decorator(func):
+        @wraps(func)
+        def wrapper(*args, **kwargs):
+            if cache_key in CACHE and CACHE[cache_key].get("expiry") - time.time() < TTL:
+                return CACHE[cache_key]["data"]
+
+            result: pd.DataFrame = func(*args, **kwargs)
+            if result is not None and not result.empty:
+                d = {"expiry": time.time(), "data": result}
+                CACHE[cache_key] = d
+                return result
+
+            CACHE[cache_key]["expiry"] += TTL
+            return CACHE[cache_key]["data"]
+
+        return wrapper
+
+    return decorator
+
+
+def cache_dict_with_custom_key(cache_key: str):
+    def decorator(func):
+        @wraps(func)
+        def wrapper(*args, **kwargs):
+            if cache_key in CACHE and CACHE[cache_key].get("expiry") - time.time() < TTL:
+                return CACHE[cache_key]["data"]
+
+            result: dict = func(*args, **kwargs)
+            if result:
+                d = {"expiry": time.time(), "data": result}
+                CACHE[cache_key] = d
+                return result
+
+            CACHE[cache_key]["expiry"] += TTL
+            return CACHE[cache_key]["data"]
+
+        return wrapper
+
+    return decorator
+
+
+if __name__ == '__main__':
+    a = time.time()
+    time.sleep(5)
+    print(time.time() - a)

utils/http_utils.py

@@ -0,0 +1,5 @@
+import requests
+
+
+def get(url: str, params: str = None, verify: bool = False):
+    return requests.get(url, params, verify=verify)