v0

README.md

@@ -11,9 +11,75 @@ pinned: false
 short_description: Streamlit template space
 ---
 
-# Welcome to Streamlit!
+# Grounding Benchmark Leaderboard Viewer
 
-Edit `/src/streamlit_app.py` to customize this app to your heart's desire. :heart:
+A Streamlit application for visualizing model performance on grounding benchmarks.
 
-If you have any questions, checkout our [documentation](https://docs.streamlit.io) and [community
-forums](https://discuss.streamlit.io).
+## Features
+
+- **Real-time Data**: Fetches results directly from the HuggingFace leaderboard repository
+- **Interactive Visualizations**: Bar charts comparing model performance across different metrics
+- **Baseline Comparisons**: Shows baseline models (Qwen2-VL, UI-TARS) alongside evaluated models
+- **UI Type Breakdown**: For ScreenSpot datasets, shows performance by:
+  - Desktop vs Web
+  - Text vs Icon elements
+  - Overall averages
+- **Model Details**: View training loss, checkpoint steps, and evaluation timestamps
+- **Raw Data Access**: Inspect the complete evaluation results JSON
+
+## Installation
+
+1. Clone or download this directory
+2. Install dependencies:
+   ```bash
+   pip install -r requirements.txt
+   ```
+
+## Running the App
+
+```bash
+streamlit run src/streamlit_app.py
+```
+
+The app will open in your browser at `http://localhost:8501`
+
+## Usage
+
+1. **Select Dataset**: Use the sidebar to choose which benchmark dataset to view (e.g., screenspot-v2, screenspot-pro)
+
+2. **Filter Models**: Optionally filter to view a specific model or all models
+
+3. **View Charts**: The main page displays:
+   - Overall metrics (number of models, best accuracy, total samples)
+   - Bar charts comparing performance across different UI types
+   - Baseline model comparisons (shown in orange)
+
+4. **Explore Details**: 
+   - Expand "Model Details" to see training metadata
+   - Expand "Detailed UI Type Breakdown" for a comprehensive table
+   - Expand "Raw Data" to inspect the complete JSON results
+
+## Data Source
+
+The app fetches data from the HuggingFace dataset repository:
+- Repository: `mlfoundations-cua-dev/leaderboard`
+- Path: `grounding/[dataset_name]/[model_results].json`
+
+## Supported Datasets
+
+- **ScreenSpot-v2**: Web and desktop UI element grounding
+- **ScreenSpot-Pro**: Professional UI grounding benchmark
+- **ShowdownClicks**: Click prediction benchmark
+- And more as they are added to the leaderboard
+
+## Baseline Models
+
+For ScreenSpot-v2, the following baselines are included:
+- Qwen2-VL-7B
+- UI-TARS-2B
+- UI-TARS-7B
+- UI-TARS-72B
+
+## Caching
+
+Results are cached for 5 minutes to improve performance. The cache automatically refreshes to show new evaluation results.

requirements.txt

@@ -1,3 +1,5 @@
-altair
-pandas
-streamlit
+streamlit>=1.28.0
+pandas>=1.5.0
+altair>=5.0.0
+huggingface-hub>=0.19.0
+numpy>=1.24.0

src/streamlit_app.py

@@ -1,40 +1,354 @@
+import streamlit as st
+import pandas as pd
 import altair as alt
+from huggingface_hub import HfApi, hf_hub_download
+import json
+from pathlib import Path
+import os
+from typing import Dict, List, Optional
 import numpy as np
-import pandas as pd
-import streamlit as st
 
-"""
-# Welcome to Streamlit!
-
-Edit `/streamlit_app.py` to customize this app to your heart's desire :heart:.
-If you have any questions, checkout our [documentation](https://docs.streamlit.io) and [community
-forums](https://discuss.streamlit.io).
-
-In the meantime, below is an example of what you can do with just a few lines of code:
-"""
-
-num_points = st.slider("Number of points in spiral", 1, 10000, 1100)
-num_turns = st.slider("Number of turns in spiral", 1, 300, 31)
-
-indices = np.linspace(0, 1, num_points)
-theta = 2 * np.pi * num_turns * indices
-radius = indices
-
-x = radius * np.cos(theta)
-y = radius * np.sin(theta)
-
-df = pd.DataFrame({
-    "x": x,
-    "y": y,
-    "idx": indices,
-    "rand": np.random.randn(num_points),
-})
-
-st.altair_chart(alt.Chart(df, height=700, width=700)
-    .mark_point(filled=True)
-    .encode(
-        x=alt.X("x", axis=None),
-        y=alt.Y("y", axis=None),
-        color=alt.Color("idx", legend=None, scale=alt.Scale()),
-        size=alt.Size("rand", legend=None, scale=alt.Scale(range=[1, 150])),
-    ))
+# Page config
+st.set_page_config(
+    page_title="Grounding Benchmark Leaderboard",
+    page_icon="🎯",
+    layout="wide"
+)
+
+# Constants
+REPO_ID = "mlfoundations-cua-dev/leaderboard"
+GROUNDING_PATH = "grounding"
+
+# Baselines for different datasets
+BASELINES = {
+    "screenspot-v2": {
+        "Qwen2-VL-7B": {
+            "desktop_text": 52.01,
+            "desktop_icon": 44.98,
+            "web_text": 33.04,
+            "web_icon": 21.84,
+            "overall": 37.96
+        },
+        "UI-TARS-2B": {
+            "desktop_text": 90.7,
+            "desktop_icon": 68.6,
+            "web_text": 87.2,
+            "web_icon": 84.7,
+            "overall": 82.8
+        },
+        "UI-TARS-7B": {
+            "desktop_text": 95.4,
+            "desktop_icon": 87.8,
+            "web_text": 93.8,
+            "web_icon": 91.6,
+            "overall": 92.2
+        },
+        "UI-TARS-72B": {
+            "desktop_text": 91.2,
+            "desktop_icon": 87.8,
+            "web_text": 87.7,
+            "web_icon": 86.3,
+            "overall": 88.3
+        }
+    }
+}
+
+@st.cache_data(ttl=300)  # Cache for 5 minutes
+def fetch_leaderboard_data():
+    """Fetch all grounding results from HuggingFace leaderboard."""
+    api = HfApi()
+    
+    try:
+        # List all files in the grounding directory
+        files = api.list_repo_files(repo_id=REPO_ID, repo_type="dataset")
+        grounding_files = [f for f in files if f.startswith(f"{GROUNDING_PATH}/") and f.endswith(".json")]
+        
+        results = []
+        for file_path in grounding_files:
+            try:
+                # Download and parse each JSON file
+                local_path = hf_hub_download(
+                    repo_id=REPO_ID,
+                    filename=file_path,
+                    repo_type="dataset"
+                )
+                
+                with open(local_path, 'r') as f:
+                    data = json.load(f)
+                
+                # Extract key information
+                metadata = data.get("metadata", {})
+                metrics = data.get("metrics", {})
+                detailed_results = data.get("detailed_results", {})
+                
+                # Parse the file path to get dataset and model info
+                path_parts = file_path.split('/')
+                dataset_name = path_parts[1] if len(path_parts) > 1 else "unknown"
+                
+                # Get model name from metadata or path
+                model_name = metadata.get("model_checkpoint", "").split('/')[-1]
+                if not model_name and len(path_parts) > 2:
+                    model_name = path_parts[2].replace("results_", "").replace(".json", "")
+                
+                # Extract UI type results if available
+                ui_type_results = detailed_results.get("by_ui_type", {})
+                dataset_type_results = detailed_results.get("by_dataset_type", {})
+                
+                results.append({
+                    "dataset": dataset_name,
+                    "model": model_name,
+                    "model_path": metadata.get("model_checkpoint", ""),
+                    "overall_accuracy": metrics.get("accuracy", 0) * 100,  # Convert to percentage
+                    "total_samples": metrics.get("total", 0),
+                    "timestamp": metadata.get("evaluation_timestamp", ""),
+                    "checkpoint_steps": metadata.get("checkpoint_steps"),
+                    "training_loss": metadata.get("training_loss"),
+                    "ui_type_results": ui_type_results,
+                    "dataset_type_results": dataset_type_results,
+                    "raw_data": data
+                })
+                
+            except Exception as e:
+                st.warning(f"Error loading {file_path}: {str(e)}")
+                continue
+        
+        return pd.DataFrame(results)
+    
+    except Exception as e:
+        st.error(f"Error fetching leaderboard data: {str(e)}")
+        return pd.DataFrame()
+
+def parse_ui_type_metrics(df: pd.DataFrame, dataset_filter: str) -> pd.DataFrame:
+    """Parse UI type metrics from the results dataframe."""
+    metrics_list = []
+    
+    for _, row in df.iterrows():
+        if row['dataset'] != dataset_filter:
+            continue
+            
+        model = row['model']
+        ui_results = row['ui_type_results']
+        
+        # For ScreenSpot datasets, we have desktop/web and text/icon
+        if 'screenspot' in dataset_filter.lower():
+            # Calculate aggregated metrics
+            desktop_text = ui_results.get('desktop_text', {}).get('correct', 0) / max(ui_results.get('desktop_text', {}).get('total', 1), 1) * 100
+            desktop_icon = ui_results.get('desktop_icon', {}).get('correct', 0) / max(ui_results.get('desktop_icon', {}).get('total', 1), 1) * 100
+            web_text = ui_results.get('web_text', {}).get('correct', 0) / max(ui_results.get('web_text', {}).get('total', 1), 1) * 100
+            web_icon = ui_results.get('web_icon', {}).get('correct', 0) / max(ui_results.get('web_icon', {}).get('total', 1), 1) * 100
+            
+            # Calculate averages
+            desktop_avg = (desktop_text + desktop_icon) / 2 if desktop_text or desktop_icon else 0
+            web_avg = (web_text + web_icon) / 2 if web_text or web_icon else 0
+            text_avg = (desktop_text + web_text) / 2 if desktop_text or web_text else 0
+            icon_avg = (desktop_icon + web_icon) / 2 if desktop_icon or web_icon else 0
+            
+            metrics_list.append({
+                'model': model,
+                'desktop_text': desktop_text,
+                'desktop_icon': desktop_icon,
+                'web_text': web_text,
+                'web_icon': web_icon,
+                'desktop_avg': desktop_avg,
+                'web_avg': web_avg,
+                'text_avg': text_avg,
+                'icon_avg': icon_avg,
+                'overall': row['overall_accuracy']
+            })
+    
+    return pd.DataFrame(metrics_list)
+
+def create_bar_chart(data: pd.DataFrame, metric: str, title: str):
+    """Create a bar chart for a specific metric."""
+    # Prepare data for the chart
+    chart_data = []
+    
+    # Add model results
+    for _, row in data.iterrows():
+        if metric in row and row[metric] > 0:
+            chart_data.append({
+                'Model': row['model'],
+                'Score': row[metric],
+                'Type': 'Evaluated'
+            })
+    
+    # Add baselines if available
+    dataset = st.session_state.get('selected_dataset', '')
+    if dataset in BASELINES:
+        for baseline_name, baseline_metrics in BASELINES[dataset].items():
+            metric_key = metric.replace('_avg', '').replace('avg', 'overall')
+            if metric_key in baseline_metrics:
+                chart_data.append({
+                    'Model': baseline_name,
+                    'Score': baseline_metrics[metric_key],
+                    'Type': 'Baseline'
+                })
+    
+    if not chart_data:
+        return None
+    
+    df_chart = pd.DataFrame(chart_data)
+    
+    # Create the bar chart
+    chart = alt.Chart(df_chart).mark_bar().encode(
+        x=alt.X('Model:N', 
+                sort=alt.EncodingSortField(field='Score', order='descending'),
+                axis=alt.Axis(labelAngle=-45)),
+        y=alt.Y('Score:Q', 
+                scale=alt.Scale(domain=[0, 100]),
+                axis=alt.Axis(title='Score (%)')),
+        color=alt.Color('Type:N', 
+                       scale=alt.Scale(domain=['Evaluated', 'Baseline'],
+                                     range=['#4ECDC4', '#FFA726'])),
+        tooltip=['Model', 'Score', 'Type']
+    ).properties(
+        title=title,
+        width=400,
+        height=300
+    )
+    
+    # Add value labels
+    text = chart.mark_text(
+        align='center',
+        baseline='bottom',
+        dy=-5
+    ).encode(
+        text=alt.Text('Score:Q', format='.1f')
+    )
+    
+    return chart + text
+
+def main():
+    st.title("🎯 Grounding Benchmark Leaderboard")
+    st.markdown("Visualization of model performance on grounding benchmarks")
+    
+    # Fetch data
+    with st.spinner("Loading leaderboard data..."):
+        df = fetch_leaderboard_data()
+    
+    if df.empty:
+        st.warning("No data available in the leaderboard.")
+        return
+    
+    # Sidebar filters
+    st.sidebar.header("Filters")
+    
+    # Dataset filter
+    datasets = sorted(df['dataset'].unique())
+    selected_dataset = st.sidebar.selectbox("Select Dataset", datasets)
+    st.session_state['selected_dataset'] = selected_dataset
+    
+    # Filter data
+    filtered_df = df[df['dataset'] == selected_dataset]
+    
+    # Model filter (optional)
+    models = ['All'] + sorted(filtered_df['model'].unique())
+    selected_model = st.sidebar.selectbox("Select Model", models)
+    
+    if selected_model != 'All':
+        filtered_df = filtered_df[filtered_df['model'] == selected_model]
+    
+    # Main content
+    st.header(f"Results for {selected_dataset}")
+    
+    # Overall metrics
+    col1, col2, col3 = st.columns(3)
+    with col1:
+        st.metric("Models Evaluated", len(filtered_df))
+    with col2:
+        if not filtered_df.empty:
+            best_acc = filtered_df['overall_accuracy'].max()
+            best_model = filtered_df[filtered_df['overall_accuracy'] == best_acc]['model'].iloc[0]
+            st.metric("Best Overall Accuracy", f"{best_acc:.1f}%", help=f"Model: {best_model}")
+    with col3:
+        total_samples = filtered_df['total_samples'].sum()
+        st.metric("Total Samples Evaluated", f"{total_samples:,}")
+    
+    # Parse UI type metrics
+    ui_metrics_df = parse_ui_type_metrics(filtered_df, selected_dataset)
+    
+    if not ui_metrics_df.empty and 'screenspot' in selected_dataset.lower():
+        st.subheader("Performance by UI Type")
+        
+        # Create charts in a grid
+        col1, col2 = st.columns(2)
+        
+        with col1:
+            # Overall Average
+            chart = create_bar_chart(ui_metrics_df, 'overall', 'Overall Average')
+            if chart:
+                st.altair_chart(chart, use_container_width=True)
+            
+            # Desktop Average
+            chart = create_bar_chart(ui_metrics_df, 'desktop_avg', 'Desktop Average')
+            if chart:
+                st.altair_chart(chart, use_container_width=True)
+            
+            # Text Average
+            chart = create_bar_chart(ui_metrics_df, 'text_avg', 'Text Average (UI-Type)')
+            if chart:
+                st.altair_chart(chart, use_container_width=True)
+        
+        with col2:
+            # Web Average
+            chart = create_bar_chart(ui_metrics_df, 'web_avg', 'Web Average')
+            if chart:
+                st.altair_chart(chart, use_container_width=True)
+            
+            # Icon Average
+            chart = create_bar_chart(ui_metrics_df, 'icon_avg', 'Icon Average (UI-Type)')
+            if chart:
+                st.altair_chart(chart, use_container_width=True)
+        
+        # Detailed breakdown
+        with st.expander("Detailed UI Type Breakdown"):
+            # Create a heatmap-style table
+            detailed_metrics = []
+            for _, row in ui_metrics_df.iterrows():
+                detailed_metrics.append({
+                    'Model': row['model'],
+                    'Desktop Text': f"{row['desktop_text']:.1f}%",
+                    'Desktop Icon': f"{row['desktop_icon']:.1f}%",
+                    'Web Text': f"{row['web_text']:.1f}%",
+                    'Web Icon': f"{row['web_icon']:.1f}%",
+                    'Overall': f"{row['overall']:.1f}%"
+                })
+            
+            if detailed_metrics:
+                st.dataframe(pd.DataFrame(detailed_metrics), use_container_width=True)
+    
+    else:
+        # For non-ScreenSpot datasets, show a simple bar chart
+        st.subheader("Model Performance")
+        
+        chart_data = filtered_df[['model', 'overall_accuracy']].copy()
+        chart_data.columns = ['Model', 'Accuracy']
+        
+        chart = alt.Chart(chart_data).mark_bar().encode(
+            x=alt.X('Model:N', sort='-y', axis=alt.Axis(labelAngle=-45)),
+            y=alt.Y('Accuracy:Q', scale=alt.Scale(domain=[0, 100])),
+            tooltip=['Model', 'Accuracy']
+        ).properties(
+            width=800,
+            height=400
+        )
+        
+        st.altair_chart(chart, use_container_width=True)
+    
+    # Model details table
+    with st.expander("Model Details"):
+        display_df = filtered_df[['model', 'overall_accuracy', 'total_samples', 'checkpoint_steps', 'training_loss', 'timestamp']].copy()
+        display_df.columns = ['Model', 'Accuracy (%)', 'Samples', 'Checkpoint Steps', 'Training Loss', 'Timestamp']
+        display_df['Accuracy (%)'] = display_df['Accuracy (%)'].apply(lambda x: f"{x:.2f}")
+        display_df['Training Loss'] = display_df['Training Loss'].apply(lambda x: f"{x:.4f}" if pd.notna(x) else "N/A")
+        st.dataframe(display_df, use_container_width=True)
+    
+    # Raw data viewer
+    with st.expander("Raw Data"):
+        if selected_model != 'All' and len(filtered_df) == 1:
+            st.json(filtered_df.iloc[0]['raw_data'])
+        else:
+            st.info("Select a specific model to view raw data")
+
+if __name__ == "__main__":
+    main()