{
 "cells": [
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-11T16:39:14.401092Z",
     "start_time": "2025-03-11T16:39:14.399279Z"
    }
   },
   "cell_type": "code",
   "source": [
    "''' Extract data from file by the special keyword '''\n",
    "import pandas as pd\n",
    "import glob\n",
    "import re\n",
    "import numpy as np\n",
    "import scipy.stats as stats\n",
    "from scipy.stats import f_oneway"
   ],
   "id": "3db8dc37abb935af",
   "outputs": [],
   "execution_count": 142
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-11T16:39:14.420609Z",
     "start_time": "2025-03-11T16:39:14.417506Z"
    }
   },
   "cell_type": "code",
   "source": [
    "def read_log(file, regression_name):\n",
    "    line_list = []\n",
    "    with open(file) as f:\n",
    "        while True:\n",
    "            lines = f.readline()\n",
    "            line_list.append(lines)\n",
    "            if not lines:\n",
    "                break\n",
    "    # print(line_list)\n",
    "    if regression_name == 'mlp':\n",
    "        keyline = \"INFO - Average testing results among all repeated 80-20 holdouts:\\n\"\n",
    "    elif regression_name == 'svr':\n",
    "        keyline = \"Average testing results among all repeated 80-20 holdouts:\\n\"\n",
    "    for i in range(len(line_list)):\n",
    "        if line_list[i] == keyline:\n",
    "            key_index = i\n",
    "    srcc = line_list[key_index + 1]\n",
    "    krcc = line_list[key_index + 2]\n",
    "    plcc = line_list[key_index + 3]\n",
    "    rmse = line_list[key_index + 4]\n",
    "    return srcc, krcc, plcc, rmse\n",
    "\n",
    "def get_value(coef, model_name, decimals):\n",
    "    if model_name == 'mlp':\n",
    "        pattern = r\"INFO - (\\w+): (\\d+\\.\\d+) \\(std: (\\d+\\.\\d+)\\)\"\n",
    "    elif model_name == 'svr':\n",
    "        pattern = r\"(\\w+): (\\d+\\.\\d+) \\(std: (\\d+\\.\\d+)\\)\"\n",
    "\n",
    "    matches = re.findall(pattern, coef)\n",
    "    for match in matches:\n",
    "        metric, value, std_dev = match\n",
    "        value = format(float(value), f'.{decimals}f')\n",
    "        std_dev = format(float(std_dev), f'.{decimals}f')\n",
    "        plusmius = u\"\\u00B1\"\n",
    "        coef_value = f\"{value} ({plusmius}{std_dev})\"\n",
    "    return coef_value\n"
   ],
   "id": "initial_id",
   "outputs": [],
   "execution_count": 143
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-11T16:39:14.447744Z",
     "start_time": "2025-03-11T16:39:14.444612Z"
    }
   },
   "cell_type": "code",
   "source": [
    "def process_log_files(data_name, model_names, regression_name, log_path, decimals):\n",
    "    df_results = pd.DataFrame(columns=['DATASET', 'MODEL', 'SRCC (±STD)', 'KRCC (±STD)', 'PLCC (±STD)', 'RMSE (±STD)'])\n",
    "    data_list, model_list, srcc_list, krcc_list, plcc_list, rmse_list = [], [], [], [], [], []\n",
    "    \n",
    "    for model in model_names:\n",
    "        log_file = f'{log_path}{data_name}_{model}_{regression_name}.log'\n",
    "        try:\n",
    "            srcc, krcc, plcc, rmse = read_log(log_file, regression_name)\n",
    "            srcc = get_value(srcc, regression_name, decimals)\n",
    "            krcc = get_value(krcc, regression_name, decimals)\n",
    "            plcc = get_value(plcc, regression_name, decimals)\n",
    "            rmse = get_value(rmse, regression_name, decimals)\n",
    "            \n",
    "            data_list.append(data_name)\n",
    "            model_list.append(model)\n",
    "            srcc_list.append(srcc)\n",
    "            krcc_list.append(krcc)\n",
    "            plcc_list.append(plcc)\n",
    "            rmse_list.append(rmse)\n",
    "        \n",
    "        except FileNotFoundError:\n",
    "            print(f\"FileNotFoundError: No such file or directory for {model}\")\n",
    "        except Exception as e:\n",
    "            print(f\"Error occurred while processing {model}: {str(e)}\")\n",
    "\n",
    "    df_results['DATASET'] = data_list\n",
    "    df_results['MODEL'] = model_list\n",
    "    df_results['SRCC (±STD)'] = srcc_list\n",
    "    df_results['KRCC (±STD)'] = krcc_list\n",
    "    df_results['PLCC (±STD)'] = plcc_list\n",
    "    df_results['RMSE (±STD)'] = rmse_list\n",
    "    \n",
    "    return df_results"
   ],
   "id": "7e9a39184bac25a7",
   "outputs": [],
   "execution_count": 144
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": "Performance comparison of the evaluated NR-VQA models",
   "id": "65b1cb6929ece5aa"
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-11T16:39:14.454239Z",
     "start_time": "2025-03-11T16:39:14.448711Z"
    }
   },
   "cell_type": "code",
   "source": [
    "data_name = 'cvd_2014'\n",
    "model_names = ['brisque', 'vbliinds', 'tlvqm', 'videval', 'rapique']\n",
    "regression_name = 'svr'\n",
    "log_path = './reported_results/'\n",
    "df_results = process_log_files(data_name, model_names, regression_name, log_path, decimals=4)\n",
    "df_results\n",
    "# csv_name = f'./csv/{data_name}_svrlog.csv'\n",
    "# df_results.to_csv(csv_name, index=None, encoding=\"UTF-8\")\n",
    "# xlsx_name = f'./save_csv/{data_name}_{regression_name}_log.xlsx'\n",
    "# df_results.to_excel(xlsx_name, index=False, encoding=\"utf-8\")"
   ],
   "id": "2ca72decdd62d8ab",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "    DATASET     MODEL       SRCC (±STD)       KRCC (±STD)       PLCC (±STD)  \\\n",
       "0  cvd_2014   brisque  0.5553 (±0.0231)  0.3895 (±0.0161)  0.5527 (±0.0627)   \n",
       "1  cvd_2014  vbliinds  0.7306 (±0.0511)  0.5393 (±0.0473)  0.7853 (±0.0414)   \n",
       "2  cvd_2014     tlvqm  0.5399 (±0.0352)  0.4006 (±0.0298)  0.5785 (±0.0498)   \n",
       "3  cvd_2014   videval  0.7663 (±0.0214)  0.5634 (±0.0221)  0.8062 (±0.0216)   \n",
       "4  cvd_2014   rapique  0.8530 (±0.0301)  0.6836 (±0.0357)  0.8766 (±0.0326)   \n",
       "\n",
       "         RMSE (±STD)  \n",
       "0  18.4752 (±1.0274)  \n",
       "1  13.7267 (±1.2130)  \n",
       "2  18.0832 (±0.7052)  \n",
       "3  13.1151 (±0.6579)  \n",
       "4  10.6670 (±1.1939)  "
      ],
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>DATASET</th>\n",
       "      <th>MODEL</th>\n",
       "      <th>SRCC (±STD)</th>\n",
       "      <th>KRCC (±STD)</th>\n",
       "      <th>PLCC (±STD)</th>\n",
       "      <th>RMSE (±STD)</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>cvd_2014</td>\n",
       "      <td>brisque</td>\n",
       "      <td>0.5553 (±0.0231)</td>\n",
       "      <td>0.3895 (±0.0161)</td>\n",
       "      <td>0.5527 (±0.0627)</td>\n",
       "      <td>18.4752 (±1.0274)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>cvd_2014</td>\n",
       "      <td>vbliinds</td>\n",
       "      <td>0.7306 (±0.0511)</td>\n",
       "      <td>0.5393 (±0.0473)</td>\n",
       "      <td>0.7853 (±0.0414)</td>\n",
       "      <td>13.7267 (±1.2130)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>cvd_2014</td>\n",
       "      <td>tlvqm</td>\n",
       "      <td>0.5399 (±0.0352)</td>\n",
       "      <td>0.4006 (±0.0298)</td>\n",
       "      <td>0.5785 (±0.0498)</td>\n",
       "      <td>18.0832 (±0.7052)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>cvd_2014</td>\n",
       "      <td>videval</td>\n",
       "      <td>0.7663 (±0.0214)</td>\n",
       "      <td>0.5634 (±0.0221)</td>\n",
       "      <td>0.8062 (±0.0216)</td>\n",
       "      <td>13.1151 (±0.6579)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>cvd_2014</td>\n",
       "      <td>rapique</td>\n",
       "      <td>0.8530 (±0.0301)</td>\n",
       "      <td>0.6836 (±0.0357)</td>\n",
       "      <td>0.8766 (±0.0326)</td>\n",
       "      <td>10.6670 (±1.1939)</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ]
     },
     "execution_count": 145,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 145
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-11T16:39:14.460774Z",
     "start_time": "2025-03-11T16:39:14.454807Z"
    }
   },
   "cell_type": "code",
   "source": [
    "data_name = 'konvid_1k'\n",
    "model_names = ['brisque', 'vbliinds', 'tlvqm', 'videval', 'rapique']\n",
    "regression_name = 'svr'\n",
    "log_path = './reported_results/'\n",
    "df_results = process_log_files(data_name, model_names, regression_name, log_path, decimals=4)\n",
    "df_results"
   ],
   "id": "42997a595fb2bca8",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "     DATASET     MODEL       SRCC (±STD)       KRCC (±STD)       PLCC (±STD)  \\\n",
       "0  konvid_1k   brisque  0.6781 (±0.0083)  0.4935 (±0.0091)  0.6746 (±0.0069)   \n",
       "1  konvid_1k  vbliinds  0.7258 (±0.0115)  0.5322 (±0.0109)  0.7155 (±0.0107)   \n",
       "2  konvid_1k     tlvqm  0.7616 (±0.0067)  0.5635 (±0.0067)  0.7463 (±0.0065)   \n",
       "3  konvid_1k   videval  0.8073 (±0.0174)  0.6036 (±0.0165)  0.7923 (±0.0169)   \n",
       "4  konvid_1k   rapique  0.8219 (±0.0057)  0.6264 (±0.0074)  0.8191 (±0.0079)   \n",
       "\n",
       "        RMSE (±STD)  \n",
       "0  0.4753 (±0.0040)  \n",
       "1  0.4498 (±0.0071)  \n",
       "2  0.4285 (±0.0047)  \n",
       "3  0.3928 (±0.0133)  \n",
       "4  0.3693 (±0.0070)  "
      ],
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>DATASET</th>\n",
       "      <th>MODEL</th>\n",
       "      <th>SRCC (±STD)</th>\n",
       "      <th>KRCC (±STD)</th>\n",
       "      <th>PLCC (±STD)</th>\n",
       "      <th>RMSE (±STD)</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>konvid_1k</td>\n",
       "      <td>brisque</td>\n",
       "      <td>0.6781 (±0.0083)</td>\n",
       "      <td>0.4935 (±0.0091)</td>\n",
       "      <td>0.6746 (±0.0069)</td>\n",
       "      <td>0.4753 (±0.0040)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>konvid_1k</td>\n",
       "      <td>vbliinds</td>\n",
       "      <td>0.7258 (±0.0115)</td>\n",
       "      <td>0.5322 (±0.0109)</td>\n",
       "      <td>0.7155 (±0.0107)</td>\n",
       "      <td>0.4498 (±0.0071)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>konvid_1k</td>\n",
       "      <td>tlvqm</td>\n",
       "      <td>0.7616 (±0.0067)</td>\n",
       "      <td>0.5635 (±0.0067)</td>\n",
       "      <td>0.7463 (±0.0065)</td>\n",
       "      <td>0.4285 (±0.0047)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>konvid_1k</td>\n",
       "      <td>videval</td>\n",
       "      <td>0.8073 (±0.0174)</td>\n",
       "      <td>0.6036 (±0.0165)</td>\n",
       "      <td>0.7923 (±0.0169)</td>\n",
       "      <td>0.3928 (±0.0133)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>konvid_1k</td>\n",
       "      <td>rapique</td>\n",
       "      <td>0.8219 (±0.0057)</td>\n",
       "      <td>0.6264 (±0.0074)</td>\n",
       "      <td>0.8191 (±0.0079)</td>\n",
       "      <td>0.3693 (±0.0070)</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ]
     },
     "execution_count": 146,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 146
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-11T16:39:14.466743Z",
     "start_time": "2025-03-11T16:39:14.461700Z"
    }
   },
   "cell_type": "code",
   "source": [
    "data_name = 'live_vqc'\n",
    "model_names = ['brisque', 'vbliinds', 'tlvqm', 'videval', 'rapique']\n",
    "regression_name = 'svr'\n",
    "log_path = './reported_results/'\n",
    "df_results = process_log_files(data_name, model_names, regression_name, log_path, decimals=4)\n",
    "df_results"
   ],
   "id": "ccdd1ce4bc21ed69",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "    DATASET     MODEL       SRCC (±STD)       KRCC (±STD)       PLCC (±STD)  \\\n",
       "0  live_vqc   brisque  0.6096 (±0.0133)  0.4420 (±0.0127)  0.6652 (±0.0176)   \n",
       "1  live_vqc  vbliinds  0.6925 (±0.0103)  0.5046 (±0.0111)  0.6872 (±0.0124)   \n",
       "2  live_vqc     tlvqm  0.8133 (±0.0104)  0.6231 (±0.0105)  0.7912 (±0.0148)   \n",
       "3  live_vqc   videval  0.7725 (±0.0093)  0.5874 (±0.0080)  0.7752 (±0.0211)   \n",
       "4  live_vqc   rapique  0.7328 (±0.0130)  0.5418 (±0.0134)  0.7520 (±0.0119)   \n",
       "\n",
       "         RMSE (±STD)  \n",
       "0  12.7480 (±0.2697)  \n",
       "1  12.4031 (±0.1948)  \n",
       "2  10.4409 (±0.3123)  \n",
       "3  10.7846 (±0.4082)  \n",
       "4  11.2538 (±0.2285)  "
      ],
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>DATASET</th>\n",
       "      <th>MODEL</th>\n",
       "      <th>SRCC (±STD)</th>\n",
       "      <th>KRCC (±STD)</th>\n",
       "      <th>PLCC (±STD)</th>\n",
       "      <th>RMSE (±STD)</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>live_vqc</td>\n",
       "      <td>brisque</td>\n",
       "      <td>0.6096 (±0.0133)</td>\n",
       "      <td>0.4420 (±0.0127)</td>\n",
       "      <td>0.6652 (±0.0176)</td>\n",
       "      <td>12.7480 (±0.2697)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>live_vqc</td>\n",
       "      <td>vbliinds</td>\n",
       "      <td>0.6925 (±0.0103)</td>\n",
       "      <td>0.5046 (±0.0111)</td>\n",
       "      <td>0.6872 (±0.0124)</td>\n",
       "      <td>12.4031 (±0.1948)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>live_vqc</td>\n",
       "      <td>tlvqm</td>\n",
       "      <td>0.8133 (±0.0104)</td>\n",
       "      <td>0.6231 (±0.0105)</td>\n",
       "      <td>0.7912 (±0.0148)</td>\n",
       "      <td>10.4409 (±0.3123)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>live_vqc</td>\n",
       "      <td>videval</td>\n",
       "      <td>0.7725 (±0.0093)</td>\n",
       "      <td>0.5874 (±0.0080)</td>\n",
       "      <td>0.7752 (±0.0211)</td>\n",
       "      <td>10.7846 (±0.4082)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>live_vqc</td>\n",
       "      <td>rapique</td>\n",
       "      <td>0.7328 (±0.0130)</td>\n",
       "      <td>0.5418 (±0.0134)</td>\n",
       "      <td>0.7520 (±0.0119)</td>\n",
       "      <td>11.2538 (±0.2285)</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ]
     },
     "execution_count": 147,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 147
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-11T16:39:14.472960Z",
     "start_time": "2025-03-11T16:39:14.467320Z"
    }
   },
   "cell_type": "code",
   "source": [
    "data_name = 'youtube_ugc'\n",
    "model_names = ['brisque', 'vbliinds', 'tlvqm', 'videval', 'rapique']\n",
    "regression_name = 'svr'\n",
    "log_path = './reported_results/'\n",
    "df_results = process_log_files(data_name, model_names, regression_name, log_path, decimals=4)\n",
    "df_results"
   ],
   "id": "d46b6e4cca31e163",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "       DATASET     MODEL       SRCC (±STD)       KRCC (±STD)  \\\n",
       "0  youtube_ugc   brisque  0.3517 (±0.0165)  0.2416 (±0.0119)   \n",
       "1  youtube_ugc  vbliinds  0.4783 (±0.0188)  0.3396 (±0.0143)   \n",
       "2  youtube_ugc     tlvqm  0.6802 (±0.0183)  0.4892 (±0.0168)   \n",
       "3  youtube_ugc   videval  0.7814 (±0.0049)  0.5906 (±0.0052)   \n",
       "4  youtube_ugc   rapique  0.7994 (±0.0064)  0.5989 (±0.0054)   \n",
       "\n",
       "        PLCC (±STD)       RMSE (±STD)  \n",
       "0  0.3768 (±0.0200)  0.6349 (±0.0056)  \n",
       "1  0.4967 (±0.0228)  0.5949 (±0.0094)  \n",
       "2  0.6876 (±0.0207)  0.4977 (±0.0132)  \n",
       "3  0.7929 (±0.0038)  0.4177 (±0.0033)  \n",
       "4  0.8300 (±0.0070)  0.3823 (±0.0068)  "
      ],
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>DATASET</th>\n",
       "      <th>MODEL</th>\n",
       "      <th>SRCC (±STD)</th>\n",
       "      <th>KRCC (±STD)</th>\n",
       "      <th>PLCC (±STD)</th>\n",
       "      <th>RMSE (±STD)</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>youtube_ugc</td>\n",
       "      <td>brisque</td>\n",
       "      <td>0.3517 (±0.0165)</td>\n",
       "      <td>0.2416 (±0.0119)</td>\n",
       "      <td>0.3768 (±0.0200)</td>\n",
       "      <td>0.6349 (±0.0056)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>youtube_ugc</td>\n",
       "      <td>vbliinds</td>\n",
       "      <td>0.4783 (±0.0188)</td>\n",
       "      <td>0.3396 (±0.0143)</td>\n",
       "      <td>0.4967 (±0.0228)</td>\n",
       "      <td>0.5949 (±0.0094)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>youtube_ugc</td>\n",
       "      <td>tlvqm</td>\n",
       "      <td>0.6802 (±0.0183)</td>\n",
       "      <td>0.4892 (±0.0168)</td>\n",
       "      <td>0.6876 (±0.0207)</td>\n",
       "      <td>0.4977 (±0.0132)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>youtube_ugc</td>\n",
       "      <td>videval</td>\n",
       "      <td>0.7814 (±0.0049)</td>\n",
       "      <td>0.5906 (±0.0052)</td>\n",
       "      <td>0.7929 (±0.0038)</td>\n",
       "      <td>0.4177 (±0.0033)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>youtube_ugc</td>\n",
       "      <td>rapique</td>\n",
       "      <td>0.7994 (±0.0064)</td>\n",
       "      <td>0.5989 (±0.0054)</td>\n",
       "      <td>0.8300 (±0.0070)</td>\n",
       "      <td>0.3823 (±0.0068)</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ]
     },
     "execution_count": 148,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 148
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": "vsfa performance",
   "id": "8489668427d0c619"
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-11T16:39:14.480216Z",
     "start_time": "2025-03-11T16:39:14.473866Z"
    }
   },
   "cell_type": "code",
   "source": [
    "datasets = {\n",
    "    'cvd_2014': \"./reported_results/VSFA-CVD2014-EXP0_metrics.npy\",\n",
    "    'konvid_1k': \"./reported_results/VSFA-KoNViD_1k-EXP0.npy\",\n",
    "    'live_vqc': \"./reported_results/VSFA-LIVE-VQC-EXP0_metrics.npy\",\n",
    "    'youtube_ugc': \"./reported_results/VSFA-YOUTUBE_UGC_ALL-EXP0.npy\"\n",
    "}\n",
    "data_list, srcc_list, krcc_list, plcc_list, rmse_list = [], [], [], [], []\n",
    "def load_metrics_npy(file_path, indices):\n",
    "    data = np.load(file_path, allow_pickle=True)\n",
    "    return [data[index] for index in indices]\n",
    "for dataset_name, file_path in datasets.items():\n",
    "    indices = [3, 4, 5, 6] if dataset_name in ['konvid_1k', 'youtube_ugc'] else [1, 2, 3, 4]\n",
    "    srcc, krcc, plcc, rmse = load_metrics_npy(file_path, indices)\n",
    "    data_list.append(dataset_name)\n",
    "    srcc_list.append(srcc)\n",
    "    krcc_list.append(krcc)\n",
    "    plcc_list.append(plcc)\n",
    "    rmse_list.append(rmse)\n",
    "\n",
    "df_vsfa = pd.DataFrame({\n",
    "    'DATASET': data_list,\n",
    "    'MODEL': 'VSFA',\n",
    "    'SRCC': srcc_list,\n",
    "    'KRCC': krcc_list,\n",
    "    'PLCC': plcc_list,\n",
    "    'RMSE': rmse_list\n",
    "}).round(4)\n",
    "df_vsfa"
   ],
   "id": "cca8633823b8c802",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "       DATASET MODEL    SRCC    KRCC    PLCC     RMSE\n",
       "0     cvd_2014  VSFA  0.8825  0.7179  0.8784   9.8619\n",
       "1    konvid_1k  VSFA  0.8067  0.6102  0.8182   0.4056\n",
       "2     live_vqc  VSFA  0.5898  0.4120  0.5894  15.3245\n",
       "3  youtube_ugc  VSFA  0.7857  0.5814  0.7808   0.4321"
      ],
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>DATASET</th>\n",
       "      <th>MODEL</th>\n",
       "      <th>SRCC</th>\n",
       "      <th>KRCC</th>\n",
       "      <th>PLCC</th>\n",
       "      <th>RMSE</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>cvd_2014</td>\n",
       "      <td>VSFA</td>\n",
       "      <td>0.8825</td>\n",
       "      <td>0.7179</td>\n",
       "      <td>0.8784</td>\n",
       "      <td>9.8619</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>konvid_1k</td>\n",
       "      <td>VSFA</td>\n",
       "      <td>0.8067</td>\n",
       "      <td>0.6102</td>\n",
       "      <td>0.8182</td>\n",
       "      <td>0.4056</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>live_vqc</td>\n",
       "      <td>VSFA</td>\n",
       "      <td>0.5898</td>\n",
       "      <td>0.4120</td>\n",
       "      <td>0.5894</td>\n",
       "      <td>15.3245</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>youtube_ugc</td>\n",
       "      <td>VSFA</td>\n",
       "      <td>0.7857</td>\n",
       "      <td>0.5814</td>\n",
       "      <td>0.7808</td>\n",
       "      <td>0.4321</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ]
     },
     "execution_count": 149,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 149
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": "Performance on ReLax-VQA",
   "id": "b02ee184866c506c"
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-11T16:39:14.486341Z",
     "start_time": "2025-03-11T16:39:14.480691Z"
    }
   },
   "cell_type": "code",
   "source": [
    "data_name = 'cvd_2014'\n",
    "model_names = ['relaxvqa']\n",
    "regression_name = 'mlp'\n",
    "log_path = './reported_results/'\n",
    "df_results = process_log_files(data_name, model_names, regression_name, log_path, decimals=4)\n",
    "df_results"
   ],
   "id": "eefdd17b16239b3a",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "    DATASET     MODEL       SRCC (±STD)       KRCC (±STD)       PLCC (±STD)  \\\n",
       "0  cvd_2014  relaxvqa  0.8643 (±0.0301)  0.6960 (±0.0442)  0.8895 (±0.0255)   \n",
       "\n",
       "        RMSE (±STD)  \n",
       "0  9.8185 (±1.0969)  "
      ],
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>DATASET</th>\n",
       "      <th>MODEL</th>\n",
       "      <th>SRCC (±STD)</th>\n",
       "      <th>KRCC (±STD)</th>\n",
       "      <th>PLCC (±STD)</th>\n",
       "      <th>RMSE (±STD)</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>cvd_2014</td>\n",
       "      <td>relaxvqa</td>\n",
       "      <td>0.8643 (±0.0301)</td>\n",
       "      <td>0.6960 (±0.0442)</td>\n",
       "      <td>0.8895 (±0.0255)</td>\n",
       "      <td>9.8185 (±1.0969)</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ]
     },
     "execution_count": 150,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 150
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-11T16:39:14.497193Z",
     "start_time": "2025-03-11T16:39:14.491884Z"
    }
   },
   "cell_type": "code",
   "source": [
    "data_name = 'konvid_1k'\n",
    "df_results = process_log_files(data_name, model_names, regression_name, log_path, decimals=4)\n",
    "df_results"
   ],
   "id": "9bb3deaf95e64d06",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "     DATASET     MODEL       SRCC (±STD)       KRCC (±STD)       PLCC (±STD)  \\\n",
       "0  konvid_1k  relaxvqa  0.8535 (±0.0218)  0.6594 (±0.0232)  0.8473 (±0.0214)   \n",
       "\n",
       "        RMSE (±STD)  \n",
       "0  0.3370 (±0.0185)  "
      ],
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>DATASET</th>\n",
       "      <th>MODEL</th>\n",
       "      <th>SRCC (±STD)</th>\n",
       "      <th>KRCC (±STD)</th>\n",
       "      <th>PLCC (±STD)</th>\n",
       "      <th>RMSE (±STD)</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>konvid_1k</td>\n",
       "      <td>relaxvqa</td>\n",
       "      <td>0.8535 (±0.0218)</td>\n",
       "      <td>0.6594 (±0.0232)</td>\n",
       "      <td>0.8473 (±0.0214)</td>\n",
       "      <td>0.3370 (±0.0185)</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ]
     },
     "execution_count": 151,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 151
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-11T16:39:14.527297Z",
     "start_time": "2025-03-11T16:39:14.522356Z"
    }
   },
   "cell_type": "code",
   "source": [
    "data_name = 'live_vqc'\n",
    "df_results = process_log_files(data_name, model_names, regression_name, log_path, decimals=4)\n",
    "df_results"
   ],
   "id": "3c66f4b2588f1baa",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "    DATASET     MODEL       SRCC (±STD)       KRCC (±STD)       PLCC (±STD)  \\\n",
       "0  live_vqc  relaxvqa  0.7655 (±0.0378)  0.5785 (±0.0365)  0.8079 (±0.0352)   \n",
       "\n",
       "        RMSE (±STD)  \n",
       "0  9.8596 (±0.9099)  "
      ],
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>DATASET</th>\n",
       "      <th>MODEL</th>\n",
       "      <th>SRCC (±STD)</th>\n",
       "      <th>KRCC (±STD)</th>\n",
       "      <th>PLCC (±STD)</th>\n",
       "      <th>RMSE (±STD)</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>live_vqc</td>\n",
       "      <td>relaxvqa</td>\n",
       "      <td>0.7655 (±0.0378)</td>\n",
       "      <td>0.5785 (±0.0365)</td>\n",
       "      <td>0.8079 (±0.0352)</td>\n",
       "      <td>9.8596 (±0.9099)</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ]
     },
     "execution_count": 152,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 152
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-11T16:39:14.583026Z",
     "start_time": "2025-03-11T16:39:14.577518Z"
    }
   },
   "cell_type": "code",
   "source": [
    "data_name = 'youtube_ugc'\n",
    "df_results = process_log_files(data_name, model_names, regression_name, log_path, decimals=4)\n",
    "df_results"
   ],
   "id": "fac097145800c895",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "       DATASET     MODEL       SRCC (±STD)       KRCC (±STD)  \\\n",
       "0  youtube_ugc  relaxvqa  0.8014 (±0.0196)  0.6167 (±0.0193)   \n",
       "\n",
       "        PLCC (±STD)       RMSE (±STD)  \n",
       "0  0.8204 (±0.0186)  0.3801 (±0.0177)  "
      ],
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>DATASET</th>\n",
       "      <th>MODEL</th>\n",
       "      <th>SRCC (±STD)</th>\n",
       "      <th>KRCC (±STD)</th>\n",
       "      <th>PLCC (±STD)</th>\n",
       "      <th>RMSE (±STD)</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>youtube_ugc</td>\n",
       "      <td>relaxvqa</td>\n",
       "      <td>0.8014 (±0.0196)</td>\n",
       "      <td>0.6167 (±0.0193)</td>\n",
       "      <td>0.8204 (±0.0186)</td>\n",
       "      <td>0.3801 (±0.0177)</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ]
     },
     "execution_count": 153,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 153
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": "Ablation study on fragments",
   "id": "ba6cedaad3ce5f1"
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-11T16:39:14.648748Z",
     "start_time": "2025-03-11T16:39:14.628197Z"
    }
   },
   "cell_type": "code",
   "source": [
    "data_name = 'konvid_1k'\n",
    "model_names = ['frame_diff_resnet50', 'optical_flow_resnet50', 'frame_diff_vit', 'optical_flow_vit', 'RF_frame_diff_resnet50', 'RF_optical_flow_resnet50', 'RF_frame_diff_vit', 'RF_optical_flow_vit',\n",
    "               'MF_resnet50', 'MF_vit', 'MF_resnet50_vit']\n",
    "regression_name = 'mlp'\n",
    "log_path = './reported_results/ablation_study/fragmentation/'\n",
    "df_results = process_log_files(data_name, model_names, regression_name, log_path, decimals=4)\n",
    "df_results"
   ],
   "id": "f4b41d3d64b5b4c",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "      DATASET                     MODEL       SRCC (±STD)       KRCC (±STD)  \\\n",
       "0   konvid_1k       frame_diff_resnet50  0.6526 (±0.0221)  0.4654 (±0.0170)   \n",
       "1   konvid_1k     optical_flow_resnet50  0.5793 (±0.0392)  0.4088 (±0.0298)   \n",
       "2   konvid_1k            frame_diff_vit  0.6384 (±0.0333)  0.4633 (±0.0245)   \n",
       "3   konvid_1k          optical_flow_vit  0.5692 (±0.0376)  0.4075 (±0.0293)   \n",
       "4   konvid_1k    RF_frame_diff_resnet50  0.7082 (±0.0345)  0.5168 (±0.0301)   \n",
       "5   konvid_1k  RF_optical_flow_resnet50  0.4844 (±0.0504)  0.3356 (±0.0369)   \n",
       "6   konvid_1k         RF_frame_diff_vit  0.7145 (±0.0297)  0.5221 (±0.0265)   \n",
       "7   konvid_1k       RF_optical_flow_vit  0.5032 (±0.0495)  0.3508 (±0.0360)   \n",
       "8   konvid_1k               MF_resnet50  0.6792 (±0.0424)  0.4852 (±0.0342)   \n",
       "9   konvid_1k                    MF_vit  0.7062 (±0.0298)  0.5130 (±0.0270)   \n",
       "10  konvid_1k           MF_resnet50_vit  0.7324 (±0.0340)  0.5337 (±0.0298)   \n",
       "\n",
       "         PLCC (±STD)       RMSE (±STD)  \n",
       "0   0.6525 (±0.0202)  0.4889 (±0.0153)  \n",
       "1   0.5848 (±0.0423)  0.5356 (±0.0233)  \n",
       "2   0.6454 (±0.0298)  0.4890 (±0.0162)  \n",
       "3   0.5844 (±0.0385)  0.5243 (±0.0191)  \n",
       "4   0.7073 (±0.0345)  0.4547 (±0.0185)  \n",
       "5   0.4958 (±0.0518)  0.5634 (±0.0213)  \n",
       "6   0.7226 (±0.0281)  0.4487 (±0.0153)  \n",
       "7   0.5335 (±0.0582)  0.5504 (±0.0224)  \n",
       "8   0.6788 (±0.0440)  0.4761 (±0.0254)  \n",
       "9   0.7057 (±0.0296)  0.4582 (±0.0175)  \n",
       "10  0.7291 (±0.0335)  0.4408 (±0.0216)  "
      ],
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>DATASET</th>\n",
       "      <th>MODEL</th>\n",
       "      <th>SRCC (±STD)</th>\n",
       "      <th>KRCC (±STD)</th>\n",
       "      <th>PLCC (±STD)</th>\n",
       "      <th>RMSE (±STD)</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>konvid_1k</td>\n",
       "      <td>frame_diff_resnet50</td>\n",
       "      <td>0.6526 (±0.0221)</td>\n",
       "      <td>0.4654 (±0.0170)</td>\n",
       "      <td>0.6525 (±0.0202)</td>\n",
       "      <td>0.4889 (±0.0153)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>konvid_1k</td>\n",
       "      <td>optical_flow_resnet50</td>\n",
       "      <td>0.5793 (±0.0392)</td>\n",
       "      <td>0.4088 (±0.0298)</td>\n",
       "      <td>0.5848 (±0.0423)</td>\n",
       "      <td>0.5356 (±0.0233)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>konvid_1k</td>\n",
       "      <td>frame_diff_vit</td>\n",
       "      <td>0.6384 (±0.0333)</td>\n",
       "      <td>0.4633 (±0.0245)</td>\n",
       "      <td>0.6454 (±0.0298)</td>\n",
       "      <td>0.4890 (±0.0162)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>konvid_1k</td>\n",
       "      <td>optical_flow_vit</td>\n",
       "      <td>0.5692 (±0.0376)</td>\n",
       "      <td>0.4075 (±0.0293)</td>\n",
       "      <td>0.5844 (±0.0385)</td>\n",
       "      <td>0.5243 (±0.0191)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>konvid_1k</td>\n",
       "      <td>RF_frame_diff_resnet50</td>\n",
       "      <td>0.7082 (±0.0345)</td>\n",
       "      <td>0.5168 (±0.0301)</td>\n",
       "      <td>0.7073 (±0.0345)</td>\n",
       "      <td>0.4547 (±0.0185)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>konvid_1k</td>\n",
       "      <td>RF_optical_flow_resnet50</td>\n",
       "      <td>0.4844 (±0.0504)</td>\n",
       "      <td>0.3356 (±0.0369)</td>\n",
       "      <td>0.4958 (±0.0518)</td>\n",
       "      <td>0.5634 (±0.0213)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>konvid_1k</td>\n",
       "      <td>RF_frame_diff_vit</td>\n",
       "      <td>0.7145 (±0.0297)</td>\n",
       "      <td>0.5221 (±0.0265)</td>\n",
       "      <td>0.7226 (±0.0281)</td>\n",
       "      <td>0.4487 (±0.0153)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>konvid_1k</td>\n",
       "      <td>RF_optical_flow_vit</td>\n",
       "      <td>0.5032 (±0.0495)</td>\n",
       "      <td>0.3508 (±0.0360)</td>\n",
       "      <td>0.5335 (±0.0582)</td>\n",
       "      <td>0.5504 (±0.0224)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>konvid_1k</td>\n",
       "      <td>MF_resnet50</td>\n",
       "      <td>0.6792 (±0.0424)</td>\n",
       "      <td>0.4852 (±0.0342)</td>\n",
       "      <td>0.6788 (±0.0440)</td>\n",
       "      <td>0.4761 (±0.0254)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>konvid_1k</td>\n",
       "      <td>MF_vit</td>\n",
       "      <td>0.7062 (±0.0298)</td>\n",
       "      <td>0.5130 (±0.0270)</td>\n",
       "      <td>0.7057 (±0.0296)</td>\n",
       "      <td>0.4582 (±0.0175)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>konvid_1k</td>\n",
       "      <td>MF_resnet50_vit</td>\n",
       "      <td>0.7324 (±0.0340)</td>\n",
       "      <td>0.5337 (±0.0298)</td>\n",
       "      <td>0.7291 (±0.0335)</td>\n",
       "      <td>0.4408 (±0.0216)</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ]
     },
     "execution_count": 154,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 154
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": "Ablation study on layer stack",
   "id": "df44ade0d921b4b5"
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-11T16:39:14.741037Z",
     "start_time": "2025-03-11T16:39:14.722921Z"
    }
   },
   "cell_type": "code",
   "source": [
    "data_name = 'konvid_1k'\n",
    "model_names = ['pool_vgg16', 'pool_resnet50', 'pool_vit', 'layer_stack_vgg16', 'layer_stack_resnet50', 'pool_pool_vgg16_vit', 'pool_pool_resnet50_vit', 'layer_stack_pool_vgg16_vit', 'layer_stack_pool_resnet50_vit']\n",
    "regression_name = 'mlp'\n",
    "log_path = './reported_results/ablation_study/layer_stack/'\n",
    "df_results = process_log_files(data_name, model_names, regression_name, log_path, decimals=4)\n",
    "df_results"
   ],
   "id": "49f7b819608adda7",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "     DATASET                          MODEL       SRCC (±STD)  \\\n",
       "0  konvid_1k                     pool_vgg16  0.6665 (±0.0370)   \n",
       "1  konvid_1k                  pool_resnet50  0.6751 (±0.0313)   \n",
       "2  konvid_1k                       pool_vit  0.6661 (±0.0396)   \n",
       "3  konvid_1k              layer_stack_vgg16  0.7513 (±0.0250)   \n",
       "4  konvid_1k           layer_stack_resnet50  0.7630 (±0.0239)   \n",
       "5  konvid_1k            pool_pool_vgg16_vit  0.7100 (±0.0273)   \n",
       "6  konvid_1k         pool_pool_resnet50_vit  0.7253 (±0.0259)   \n",
       "7  konvid_1k     layer_stack_pool_vgg16_vit  0.7650 (±0.0245)   \n",
       "8  konvid_1k  layer_stack_pool_resnet50_vit  0.7697 (±0.0230)   \n",
       "\n",
       "        KRCC (±STD)       PLCC (±STD)       RMSE (±STD)  \n",
       "0  0.4878 (±0.0313)  0.6880 (±0.0328)  0.4629 (±0.0188)  \n",
       "1  0.4881 (±0.0257)  0.7047 (±0.0292)  0.4550 (±0.0162)  \n",
       "2  0.4906 (±0.0332)  0.7082 (±0.0356)  0.4515 (±0.0199)  \n",
       "3  0.5626 (±0.0228)  0.7700 (±0.0219)  0.4108 (±0.0186)  \n",
       "4  0.5699 (±0.0219)  0.7828 (±0.0224)  0.3990 (±0.0162)  \n",
       "5  0.5210 (±0.0235)  0.7364 (±0.0247)  0.4370 (±0.0153)  \n",
       "6  0.5308 (±0.0223)  0.7469 (±0.0263)  0.4277 (±0.0151)  \n",
       "7  0.5742 (±0.0236)  0.7827 (±0.0213)  0.4042 (±0.0177)  \n",
       "8  0.5779 (±0.0213)  0.7897 (±0.0209)  0.3948 (±0.0152)  "
      ],
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>DATASET</th>\n",
       "      <th>MODEL</th>\n",
       "      <th>SRCC (±STD)</th>\n",
       "      <th>KRCC (±STD)</th>\n",
       "      <th>PLCC (±STD)</th>\n",
       "      <th>RMSE (±STD)</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>konvid_1k</td>\n",
       "      <td>pool_vgg16</td>\n",
       "      <td>0.6665 (±0.0370)</td>\n",
       "      <td>0.4878 (±0.0313)</td>\n",
       "      <td>0.6880 (±0.0328)</td>\n",
       "      <td>0.4629 (±0.0188)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>konvid_1k</td>\n",
       "      <td>pool_resnet50</td>\n",
       "      <td>0.6751 (±0.0313)</td>\n",
       "      <td>0.4881 (±0.0257)</td>\n",
       "      <td>0.7047 (±0.0292)</td>\n",
       "      <td>0.4550 (±0.0162)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>konvid_1k</td>\n",
       "      <td>pool_vit</td>\n",
       "      <td>0.6661 (±0.0396)</td>\n",
       "      <td>0.4906 (±0.0332)</td>\n",
       "      <td>0.7082 (±0.0356)</td>\n",
       "      <td>0.4515 (±0.0199)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>konvid_1k</td>\n",
       "      <td>layer_stack_vgg16</td>\n",
       "      <td>0.7513 (±0.0250)</td>\n",
       "      <td>0.5626 (±0.0228)</td>\n",
       "      <td>0.7700 (±0.0219)</td>\n",
       "      <td>0.4108 (±0.0186)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>konvid_1k</td>\n",
       "      <td>layer_stack_resnet50</td>\n",
       "      <td>0.7630 (±0.0239)</td>\n",
       "      <td>0.5699 (±0.0219)</td>\n",
       "      <td>0.7828 (±0.0224)</td>\n",
       "      <td>0.3990 (±0.0162)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>konvid_1k</td>\n",
       "      <td>pool_pool_vgg16_vit</td>\n",
       "      <td>0.7100 (±0.0273)</td>\n",
       "      <td>0.5210 (±0.0235)</td>\n",
       "      <td>0.7364 (±0.0247)</td>\n",
       "      <td>0.4370 (±0.0153)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>konvid_1k</td>\n",
       "      <td>pool_pool_resnet50_vit</td>\n",
       "      <td>0.7253 (±0.0259)</td>\n",
       "      <td>0.5308 (±0.0223)</td>\n",
       "      <td>0.7469 (±0.0263)</td>\n",
       "      <td>0.4277 (±0.0151)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>konvid_1k</td>\n",
       "      <td>layer_stack_pool_vgg16_vit</td>\n",
       "      <td>0.7650 (±0.0245)</td>\n",
       "      <td>0.5742 (±0.0236)</td>\n",
       "      <td>0.7827 (±0.0213)</td>\n",
       "      <td>0.4042 (±0.0177)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>konvid_1k</td>\n",
       "      <td>layer_stack_pool_resnet50_vit</td>\n",
       "      <td>0.7697 (±0.0230)</td>\n",
       "      <td>0.5779 (±0.0213)</td>\n",
       "      <td>0.7897 (±0.0209)</td>\n",
       "      <td>0.3948 (±0.0152)</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ]
     },
     "execution_count": 155,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 155
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-11T16:39:14.762824Z",
     "start_time": "2025-03-11T16:39:14.759557Z"
    }
   },
   "cell_type": "code",
   "source": [
    "def process_finetune_csv(pattern, select_criteria, is_finetune):\n",
    "    csv_files = glob.glob(pattern)\n",
    "    combined_df = pd.DataFrame()\n",
    "    if is_finetune == True:\n",
    "        for file in csv_files:\n",
    "            df = pd.read_csv(file)\n",
    "            df['LAYER_NAME'] = 'layer_stack+pool'\n",
    "            combined_df = pd.concat([combined_df, df], ignore_index=True)\n",
    "            df.to_csv(file, index=False)\n",
    "        combined_df = combined_df.sort_values(by='DATASET')\n",
    "        combined_df = combined_df.round(4)\n",
    "        df_byrmse = combined_df[combined_df['SELECT_CRITERIA'] == 'byrmse']\n",
    "        df_bykrcc = combined_df[combined_df['SELECT_CRITERIA'] == 'bykrcc']\n",
    "        if len(df_byrmse) == len(df_bykrcc):\n",
    "            combined_horizontal = pd.concat([df_byrmse.reset_index(drop=True), df_bykrcc.reset_index(drop=True)], axis=1)\n",
    "            print(f\"Performance with SELECT_CRITERIA = {select_criteria} on LSVQ trained model\")\n",
    "            return combined_horizontal\n",
    "    else:\n",
    "        for file in csv_files:\n",
    "            df = pd.read_csv(file)\n",
    "            df['LAYER_NAME'] = 'layer_stack+pool'\n",
    "            combined_df = pd.concat([combined_df, df], ignore_index=True)\n",
    "            df.to_csv(file, index=False)\n",
    "        combined_df = combined_df.sort_values(by='DATASET')\n",
    "        combined_df = combined_df.round(4)\n",
    "        return combined_df"
   ],
   "id": "8ca6931697279736",
   "outputs": [],
   "execution_count": 156
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": "with Fine-tuning on an LSVQ-trained model",
   "id": "8d249d7514f827ad"
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-11T16:39:14.828569Z",
     "start_time": "2025-03-11T16:39:14.809573Z"
    }
   },
   "cell_type": "code",
   "source": [
    "log_path = './reported_results/fine_tune/'\n",
    "pattern_rmse = log_path + \"/*_relaxvqa_byrmse_finetune.csv\"\n",
    "result = process_finetune_csv(pattern_rmse, 'byrmse', True)\n",
    "result"
   ],
   "id": "fbbf12f1692e1070",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Performance with SELECT_CRITERIA = byrmse on LSVQ trained model\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "       DATASET     MODEL        LAYER_NAME    SRCC    KRCC    PLCC    RMSE  \\\n",
       "0     cvd_2014  relaxvqa  layer_stack+pool  0.8974  0.7299  0.9294  8.1812   \n",
       "1    konvid_1k  relaxvqa  layer_stack+pool  0.8720  0.6881  0.8668  0.3211   \n",
       "2     live_vqc  relaxvqa  layer_stack+pool  0.8468  0.6649  0.8876  7.9869   \n",
       "3  youtube_ugc  relaxvqa  layer_stack+pool  0.8469  0.6623  0.8652  0.3437   \n",
       "\n",
       "  SELECT_CRITERIA      DATASET     MODEL        LAYER_NAME    SRCC    KRCC  \\\n",
       "0          byrmse     cvd_2014  relaxvqa  layer_stack+pool  0.8899  0.7169   \n",
       "1          byrmse    konvid_1k  relaxvqa  layer_stack+pool  0.8694  0.6834   \n",
       "2          byrmse     live_vqc  relaxvqa  layer_stack+pool  0.8367  0.6531   \n",
       "3          byrmse  youtube_ugc  relaxvqa  layer_stack+pool  0.8456  0.6562   \n",
       "\n",
       "     PLCC    RMSE SELECT_CRITERIA  \n",
       "0  0.9124  9.0719          bykrcc  \n",
       "1  0.8620  0.3263          bykrcc  \n",
       "2  0.8726  8.4696          bykrcc  \n",
       "3  0.8631  0.3462          bykrcc  "
      ],
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>DATASET</th>\n",
       "      <th>MODEL</th>\n",
       "      <th>LAYER_NAME</th>\n",
       "      <th>SRCC</th>\n",
       "      <th>KRCC</th>\n",
       "      <th>PLCC</th>\n",
       "      <th>RMSE</th>\n",
       "      <th>SELECT_CRITERIA</th>\n",
       "      <th>DATASET</th>\n",
       "      <th>MODEL</th>\n",
       "      <th>LAYER_NAME</th>\n",
       "      <th>SRCC</th>\n",
       "      <th>KRCC</th>\n",
       "      <th>PLCC</th>\n",
       "      <th>RMSE</th>\n",
       "      <th>SELECT_CRITERIA</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>cvd_2014</td>\n",
       "      <td>relaxvqa</td>\n",
       "      <td>layer_stack+pool</td>\n",
       "      <td>0.8974</td>\n",
       "      <td>0.7299</td>\n",
       "      <td>0.9294</td>\n",
       "      <td>8.1812</td>\n",
       "      <td>byrmse</td>\n",
       "      <td>cvd_2014</td>\n",
       "      <td>relaxvqa</td>\n",
       "      <td>layer_stack+pool</td>\n",
       "      <td>0.8899</td>\n",
       "      <td>0.7169</td>\n",
       "      <td>0.9124</td>\n",
       "      <td>9.0719</td>\n",
       "      <td>bykrcc</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>konvid_1k</td>\n",
       "      <td>relaxvqa</td>\n",
       "      <td>layer_stack+pool</td>\n",
       "      <td>0.8720</td>\n",
       "      <td>0.6881</td>\n",
       "      <td>0.8668</td>\n",
       "      <td>0.3211</td>\n",
       "      <td>byrmse</td>\n",
       "      <td>konvid_1k</td>\n",
       "      <td>relaxvqa</td>\n",
       "      <td>layer_stack+pool</td>\n",
       "      <td>0.8694</td>\n",
       "      <td>0.6834</td>\n",
       "      <td>0.8620</td>\n",
       "      <td>0.3263</td>\n",
       "      <td>bykrcc</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>live_vqc</td>\n",
       "      <td>relaxvqa</td>\n",
       "      <td>layer_stack+pool</td>\n",
       "      <td>0.8468</td>\n",
       "      <td>0.6649</td>\n",
       "      <td>0.8876</td>\n",
       "      <td>7.9869</td>\n",
       "      <td>byrmse</td>\n",
       "      <td>live_vqc</td>\n",
       "      <td>relaxvqa</td>\n",
       "      <td>layer_stack+pool</td>\n",
       "      <td>0.8367</td>\n",
       "      <td>0.6531</td>\n",
       "      <td>0.8726</td>\n",
       "      <td>8.4696</td>\n",
       "      <td>bykrcc</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>youtube_ugc</td>\n",
       "      <td>relaxvqa</td>\n",
       "      <td>layer_stack+pool</td>\n",
       "      <td>0.8469</td>\n",
       "      <td>0.6623</td>\n",
       "      <td>0.8652</td>\n",
       "      <td>0.3437</td>\n",
       "      <td>byrmse</td>\n",
       "      <td>youtube_ugc</td>\n",
       "      <td>relaxvqa</td>\n",
       "      <td>layer_stack+pool</td>\n",
       "      <td>0.8456</td>\n",
       "      <td>0.6562</td>\n",
       "      <td>0.8631</td>\n",
       "      <td>0.3462</td>\n",
       "      <td>bykrcc</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ]
     },
     "execution_count": 157,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 157
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-11T16:39:14.844063Z",
     "start_time": "2025-03-11T16:39:14.842635Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# pattern_krcc = \"./reported_results/old/*_relaxvqa_bykrcc_finetune.csv\"\n",
    "# result = process_finetune_csv(pattern_krcc, 'bykrcc', True)\n",
    "# result"
   ],
   "id": "a3e5697b434fb2e5",
   "outputs": [],
   "execution_count": 158
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": "without Fine-tuning on an LSVQ-trained model",
   "id": "3862e848cc4a698c"
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-11T16:39:14.924048Z",
     "start_time": "2025-03-11T16:39:14.910223Z"
    }
   },
   "cell_type": "code",
   "source": [
    "pattern_rmse_wo = log_path + \"/*_relaxvqa_byrmse*_wo_finetune.csv\"\n",
    "result_wo_finetune = process_finetune_csv(pattern_rmse_wo, 'byrmse', False)\n",
    "result_wo_finetune"
   ],
   "id": "84a42c874461ba03",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "       DATASET     MODEL        LAYER_NAME    SRCC    KRCC    PLCC     RMSE  \\\n",
       "1     cvd_2014  relaxvqa  layer_stack+pool  0.7845  0.5930  0.8336  12.2445   \n",
       "0    konvid_1k  relaxvqa  layer_stack+pool  0.8312  0.6418  0.8427   0.3466   \n",
       "2     live_vqc  relaxvqa  layer_stack+pool  0.7664  0.5812  0.8242   9.8201   \n",
       "3  youtube_ugc  relaxvqa  layer_stack+pool  0.8104  0.6131  0.8354   0.3768   \n",
       "\n",
       "  SELECT_CRITERIA  \n",
       "1          byrmse  \n",
       "0          byrmse  \n",
       "2          byrmse  \n",
       "3          byrmse  "
      ],
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>DATASET</th>\n",
       "      <th>MODEL</th>\n",
       "      <th>LAYER_NAME</th>\n",
       "      <th>SRCC</th>\n",
       "      <th>KRCC</th>\n",
       "      <th>PLCC</th>\n",
       "      <th>RMSE</th>\n",
       "      <th>SELECT_CRITERIA</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>cvd_2014</td>\n",
       "      <td>relaxvqa</td>\n",
       "      <td>layer_stack+pool</td>\n",
       "      <td>0.7845</td>\n",
       "      <td>0.5930</td>\n",
       "      <td>0.8336</td>\n",
       "      <td>12.2445</td>\n",
       "      <td>byrmse</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>konvid_1k</td>\n",
       "      <td>relaxvqa</td>\n",
       "      <td>layer_stack+pool</td>\n",
       "      <td>0.8312</td>\n",
       "      <td>0.6418</td>\n",
       "      <td>0.8427</td>\n",
       "      <td>0.3466</td>\n",
       "      <td>byrmse</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>live_vqc</td>\n",
       "      <td>relaxvqa</td>\n",
       "      <td>layer_stack+pool</td>\n",
       "      <td>0.7664</td>\n",
       "      <td>0.5812</td>\n",
       "      <td>0.8242</td>\n",
       "      <td>9.8201</td>\n",
       "      <td>byrmse</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>youtube_ugc</td>\n",
       "      <td>relaxvqa</td>\n",
       "      <td>layer_stack+pool</td>\n",
       "      <td>0.8104</td>\n",
       "      <td>0.6131</td>\n",
       "      <td>0.8354</td>\n",
       "      <td>0.3768</td>\n",
       "      <td>byrmse</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ]
     },
     "execution_count": 159,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 159
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-11T16:39:14.926433Z",
     "start_time": "2025-03-11T16:39:14.924907Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# pattern_krcc_wo = \"./reported_results/old/*_relaxvqa_bykrcc*_wo_finetune.csv\"\n",
    "# result = process_finetune_csv(pattern_krcc_wo, 'bykrcc', False)\n",
    "# result_wo_finetune"
   ],
   "id": "8dfee368cceec0e1",
   "outputs": [],
   "execution_count": 160
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": "overall performance",
   "id": "6c02e6c294f8adf7"
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-11T16:39:14.984802Z",
     "start_time": "2025-03-11T16:39:14.977867Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# new\n",
    "data = {\n",
    "    'BRISQUE': {'SRCC': [0.5553, 0.6781, 0.6096, 0.3517], 'PLCC': [0.5527, 0.6746, 0.6652, 0.3768]},\n",
    "    'V-BLIINDS': {'SRCC': [0.7306, 0.7258, 0.6925, 0.4783], 'PLCC': [0.7853, 0.7155, 0.6872, 0.4967]},\n",
    "    'TLVQM': {'SRCC': [0.5399, 0.7616, 0.8133, 0.6802], 'PLCC': [0.5785, 0.7463, 0.7912, 0.6876]},\n",
    "    'VIDEVAL': {'SRCC': [0.7663, 0.8073, 0.7725, 0.7814], 'PLCC': [0.8062, 0.7923, 0.7752, 0.7929]},\n",
    "    'RAPIQUE': {'SRCC': [0.8530, 0.8219, 0.7328, 0.7994], 'PLCC': [0.8766, 0.8191, 0.7520, 0.8300]},\n",
    "    'VSFA': {'SRCC': [0.8825, 0.8067, 0.5898, 0.7857], 'PLCC': [0.8784, 0.8182, 0.5894, 0.7808]},\n",
    "    'Fast-VQA-B (w/o FT)': {'SRCC': [0.8124, 0.8581, 0.8261, 0.7241], 'PLCC': [0.8221, 0.8551, 0.8466, 0.7364]},\n",
    "    'Fast-VQA-B (w/ FT)': {'SRCC': [0.8530, 0.8869, 0.8484, 0.8385], 'PLCC': [0.8456, 0.8776, 0.8703, 0.8302]},\n",
    "    'DOVER (Technical Branch)': {'SRCC': [0.8571, 0.8736, 0.7898, 0.8181], 'PLCC': [0.8588, 0.8852, 0.8573, 0.8197]},\n",
    "    'ReLaX-VQA (ours)': {'SRCC': [0.8643, 0.8535, 0.7655, 0.8014], 'PLCC': [0.8895, 0.8473, 0.8079, 0.8204]},\n",
    "    'ReLaX-VQA (w/o FT)': {'SRCC': [0.7845, 0.8312, 0.7664, 0.8104], 'PLCC': [0.8336, 0.8427, 0.8242, 0.8354]},\n",
    "    'ReLaX-VQA (w/ FT)': {'SRCC': [0.8974, 0.8720, 0.8468, 0.8469], 'PLCC': [0.9294, 0.8668, 0.8876, 0.8652]},\n",
    "}\n",
    "# avg performance of each model for each dataset\n",
    "avg_performance = {}\n",
    "for model, metrics in data.items():\n",
    "    avg_srcc = np.mean(metrics['SRCC'])\n",
    "    avg_plcc = np.mean(metrics['PLCC'])\n",
    "    avg_performance[model] = {'SRCC': avg_srcc, 'PLCC': avg_plcc}\n",
    "df_avg_performance = pd.DataFrame.from_dict(avg_performance, orient='index').round(4)\n",
    "df_avg_performance"
   ],
   "id": "ce48ed6f16d15be6",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "                            SRCC    PLCC\n",
       "BRISQUE                   0.5487  0.5673\n",
       "V-BLIINDS                 0.6568  0.6712\n",
       "TLVQM                     0.6988  0.7009\n",
       "VIDEVAL                   0.7819  0.7916\n",
       "RAPIQUE                   0.8018  0.8194\n",
       "VSFA                      0.7662  0.7667\n",
       "Fast-VQA-B (w/o FT)       0.8052  0.8151\n",
       "Fast-VQA-B (w/ FT)        0.8567  0.8559\n",
       "DOVER (Technical Branch)  0.8347  0.8552\n",
       "ReLaX-VQA (ours)          0.8212  0.8413\n",
       "ReLaX-VQA (w/o FT)        0.7981  0.8340\n",
       "ReLaX-VQA (w/ FT)         0.8658  0.8872"
      ],
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>SRCC</th>\n",
       "      <th>PLCC</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>BRISQUE</th>\n",
       "      <td>0.5487</td>\n",
       "      <td>0.5673</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>V-BLIINDS</th>\n",
       "      <td>0.6568</td>\n",
       "      <td>0.6712</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>TLVQM</th>\n",
       "      <td>0.6988</td>\n",
       "      <td>0.7009</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>VIDEVAL</th>\n",
       "      <td>0.7819</td>\n",
       "      <td>0.7916</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>RAPIQUE</th>\n",
       "      <td>0.8018</td>\n",
       "      <td>0.8194</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>VSFA</th>\n",
       "      <td>0.7662</td>\n",
       "      <td>0.7667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Fast-VQA-B (w/o FT)</th>\n",
       "      <td>0.8052</td>\n",
       "      <td>0.8151</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Fast-VQA-B (w/ FT)</th>\n",
       "      <td>0.8567</td>\n",
       "      <td>0.8559</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DOVER (Technical Branch)</th>\n",
       "      <td>0.8347</td>\n",
       "      <td>0.8552</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ReLaX-VQA (ours)</th>\n",
       "      <td>0.8212</td>\n",
       "      <td>0.8413</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ReLaX-VQA (w/o FT)</th>\n",
       "      <td>0.7981</td>\n",
       "      <td>0.8340</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ReLaX-VQA (w/ FT)</th>\n",
       "      <td>0.8658</td>\n",
       "      <td>0.8872</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ]
     },
     "execution_count": 161,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 161
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}