MC1 column had 8 rows with a value of 1. It didn't make sense given the next highest value was 0.47 . Assuming they were data errors, they were removed

result_data_processor.py

@@ -48,6 +48,19 @@ class ResultDataProcessor:
         df.index = (df.index.str.replace('mc\|0', 'mc2', regex=True))
         df = df.loc[['harness|truthfulqa:mc2']]
         return df[[model_name]]
+    
+    # remove extreme outliers from column harness|truthfulqa:mc1
+    def _remove_mc1_outliers(self, df):
+        mc1 = df['harness|truthfulqa:mc1']
+        # Identify the outliers
+        # outliers_condition = mc1 > mc1.quantile(.95)
+        outliers_condition = mc1 == 1.0
+        # Print out the number of outliers
+        print('Number of outliers: ', outliers_condition.sum())
+        # Replace the outliers with NaN
+        df.loc[outliers_condition, 'harness|truthfulqa:mc1'] = np.nan
+        return df
+
 
     
     @staticmethod
@@ -119,6 +132,9 @@ class ResultDataProcessor:
         cols = cols[-1:] + cols[:-1]
         data = data[cols]
 
+        # remove extreme outliers from column harness|truthfulqa:mc1
+        data = self._remove_mc1_outliers(data)
+
         return data
     
     def rank_data(self):

test_data_processing.py

@@ -34,6 +34,13 @@ class TestResultDataProcessor(unittest.TestCase):
     def test_truthfulqa_mc(self):
         data = self.processor.data
         self.assertNotIn('truthfulqa:mc', data.columns)
+
+    # check for extreme outliers in mc1 column
+    def test_mc1_outliers(self):
+        data = self.processor.data
+        mc1 = data['harness|truthfulqa:mc1']
+        self.assertLess(mc1.max(), 1.0)
+        self.assertGreater(mc1.min(), 0.0)
         
 if __name__ == '__main__':
     unittest.main()