<NME> Calibration.java
<BEF> package edu.neu.ccs.pyramid.application;

import edu.neu.ccs.pyramid.calibration.*;
import edu.neu.ccs.pyramid.configuration.Config;
import edu.neu.ccs.pyramid.dataset.*;
import edu.neu.ccs.pyramid.multilabel_classification.PluginPredictor;
import edu.neu.ccs.pyramid.multilabel_classification.imlgb.*;
import edu.neu.ccs.pyramid.multilabel_classification.thresholding.TunedMarginalClassifier;
import edu.neu.ccs.pyramid.util.Serialization;
import org.apache.commons.io.FileUtils;
import org.apache.mahout.math.Vector;
import edu.neu.ccs.pyramid.regression.regression_tree.RegTreeConfig;
import edu.neu.ccs.pyramid.regression.regression_tree.RegTreeFactory;
import edu.neu.ccs.pyramid.util.*;
import org.apache.commons.io.FileUtils;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet;
import org.apache.mahout.math.DenseVector;
import org.apache.mahout.math.Vector;


import java.io.File;
import java.io.IOException;
        }

        Config config = new Config(args[0]);
        System.out.println("=========original probabilities==============");
        original(config);
        calibration(config);

    }

    private static void original(Config config) throws Exception{
        IMLGradientBoosting boosting = (IMLGradientBoosting)Serialization.deserialize(config.getString("input.model"));
        File modelFolder = (new File(config.getString("input.model"))).getParentFile();
        MultiLabelClfDataSet dataSet = TRECFormat.loadMultiLabelClfDataSet(config.getString("input.data"), DataSetType.ML_CLF_SPARSE,
                true);
        int numIntervals = 10;
        String predictTarget = config.getString("predict.Target");
        PluginPredictor<IMLGradientBoosting> pluginPredictorTmp = null;
            fileHandler.close();
        }
    }
    public static void main(String[] args) throws Exception {
        Config config = new Config(args[0]);
        main(config);

    }


    private static void calibrate(Config config, Logger logger) throws Exception{

        logger.info("start training calibrator");

        MultiLabelClfDataSet train = TRECFormat.loadMultiLabelClfDataSet(config.getString("input.trainData"), DataSetType.ML_CLF_SEQ_SPARSE, true);

        MultiLabelClfDataSet cal = TRECFormat.loadMultiLabelClfDataSet(config.getString("input.calibData"), DataSetType.ML_CLF_SEQ_SPARSE, true);

        MultiLabel[] calPred = loadPredictions(config.getString("input.calibPredictions"));

        int[] calIds = loadIds(config.getString("input.calibPredictions"));
        Vector[] calibScore = loadFeatures(config.getString("input.calibPredictions"));

        Pair<RegDataSet,PredictionFeatureExtractor> pair = createCaliData(cal,calPred,calibScore, calIds, train);
        RegDataSet calibRegData = pair.getFirst();
        PredictionFeatureExtractor predictionFeatureExtractor = pair.getSecond();
        int[] monotonicity = pair.getSecond().featureMonotonicity();

        MultiLabelClfDataSet valid = TRECFormat.loadMultiLabelClfDataSet(config.getString("input.validData"), DataSetType.ML_CLF_SEQ_SPARSE, true);

        MultiLabel[] validPred = loadPredictions(config.getString("input.validPredictions"));

        int[] validIds = loadIds(config.getString("input.validPredictions"));
        Vector[] validScore = loadFeatures(config.getString("input.validPredictions"));

        RegDataSet validRegData = createCaliData(valid,validPred,validScore, validIds, predictionFeatureExtractor).getFirst();

        LSBoost lsBoost = trainCalibrator(calibRegData,validRegData,monotonicity);

        VectorCalibrator vectorCalibrator = new RegressorCalibrator(lsBoost);

        Serialization.serialize(vectorCalibrator,Paths.get(config.getString("output.dir"),"set_calibrator").toFile());
        Serialization.serialize(predictionFeatureExtractor,Paths.get(config.getString("output.dir"),"prediction_feature_extractor").toFile());

        logger.info("finish training calibrator");
    }

    private static LSBoost trainCalibrator(RegDataSet calib, RegDataSet valid, int[] monotonicity){
        LSBoost lsBoost = new LSBoost();

        RegTreeConfig regTreeConfig = new RegTreeConfig().setMaxNumLeaves(10).setMinDataPerLeaf(5);
        RegTreeFactory regTreeFactory = new RegTreeFactory(regTreeConfig);
        LSBoostOptimizer optimizer = new LSBoostOptimizer(lsBoost, calib, regTreeFactory, calib.getLabels());
        if (true){
            int[][] mono = new int[1][calib.getNumFeatures()];
            mono[0] = monotonicity;
            optimizer.setMonotonicity(mono);
        }
        optimizer.setShrinkage(0.1);
        optimizer.initialize();
        EarlyStopper earlyStopper = new EarlyStopper(EarlyStopper.Goal.MINIMIZE,5);

    }

    private static void calibration(Config config) throws Exception{
        IMLGradientBoosting boosting = (IMLGradientBoosting)Serialization.deserialize(config.getString("input.model"));
        File modelFolder = (new File(config.getString("input.model"))).getParentFile();
        MultiLabelClfDataSet dataSet = TRECFormat.loadMultiLabelClfDataSet(config.getString("input.data"), DataSetType.ML_CLF_SPARSE,
                true);

        IMLGBPlattScaling scaling = new IMLGBPlattScaling(boosting, dataSet);

        int numIntervals = 10;
        String predictTarget = config.getString("predict.Target");
        PluginPredictor<IMLGradientBoosting> pluginPredictorTmp = null;

        switch (predictTarget){
            case "subsetAccuracy":
                pluginPredictorTmp = new SubsetAccPredictor(boosting);
                    } catch (ClassNotFoundException e) {
                        e.printStackTrace();
                    }
                }
                if (earlyStopper.shouldStop()){
                    break;
                }
            }
        }
        return bestModel;
    }


    private static MultiLabel[] loadPredictions(String file) throws Exception{
        List<String> lines = FileUtils.readLines(new File(file));
        MultiLabel[] predictions = new MultiLabel[lines.size()];
        for (int i=0;i<lines.size();i++){
            predictions[i] = new MultiLabel();
            String[] split = lines.get(i).split(Pattern.quote("("))[0].split(Pattern.quote(":"))[1].replace(" ","").split(",");
            for (String l: split){
                if (!l.isEmpty()){
                    predictions[i].addLabel(Integer.parseInt(l));
                }

            }
        }
        return predictions;
    }

        double intervalSize = 1.0/numIntervals;
        DecimalFormat decimalFormat = new DecimalFormat("#0.00");
        System.out.println("=========calibrated probabilities==============");
        System.out.println("interval"+"\t"+"total"+"\t"+"correct"+"\t\t"+"incorrect"+"\t"+"accuracy"+"\t"+"average confidence");
        for (int i=0;i<numIntervals;i++){
            double left = intervalSize*i;
            String split = lines.get(i).split(Pattern.quote("("))[1].replace(")","");
            String[] features = split.split(",");
            Vector vector = new DenseVector(features.length);
            for (int j=0;j<features.length;j++){
                vector.set(j,Double.parseDouble(features[j].trim()));
            }
            System.out.println("["+decimalFormat.format(left)+", "+decimalFormat.format(right)+"]"+"\t"+matched.size()+"\t"+numPos+"\t\t"+numNeg+"\t\t"+decimalFormat.format(accuracy)+"\t\t"+decimalFormat.format(aveProb));
        }

        dump(0.7,0.8, dataSet, config, results);
        dump(0.8,0.9, dataSet, config, results);
        dump(0.9,1, dataSet, config, results);

    }

    private static void dump(double left, double right, MultiLabelClfDataSet dataSet, Config config, List<Result> results) throws Exception{
        File data = new File(config.getString("input.data")+"_calibration_"+left+"-"+right);
        List<Integer>ids = results.stream().filter(res->res.probability>=left&&res.probability<=right).map(res->res.intId).collect(Collectors.toList());
        if (ids.size()>0){
            MultiLabelClfDataSet subset = DataSetUtil.sampleData(dataSet,ids);
            TRECFormat.save(subset, data);
            //todo this is a hack
            FileUtils.copyFile(new File(config.getString("input.data"),"data_config.json"), new File(data, "data_config.json"));
            System.out.println("saved data with calibrated probabilities between "+left+" and "+right+" to "+data.getAbsolutePath());
        }

    }

    static class Result{
        int intId;
        double probability;
        VectorCalibrator setCalibrator = (VectorCalibrator) Serialization.deserialize(Paths.get(config.getString("output.dir"),"set_calibrator").toFile());

        PredictionFeatureExtractor predictionFeatureExtractor = (PredictionFeatureExtractor) Serialization.deserialize(Paths.get(config.getString("output.dir"),"prediction_feature_extractor").toFile());

        MultiLabel[] predictions = loadPredictions(config.getString("input.testPredictions"));
        Vector[] scores = loadFeatures(config.getString("input.testPredictions"));

        int[] ids = loadIds(config.getString("input.testPredictions"));

        RegDataSet testRegData = createCaliData(test,predictions,scores, ids, predictionFeatureExtractor).getFirst();
        if (true) {
            logger.info("calibration performance on test set");

            List<CalibrationDataGenerator.CalibrationInstance> instances = IntStream.range(0, test.getNumDataPoints()).parallel()
                    .boxed().map(i -> {

                        CalibrationDataGenerator.CalibrationInstance instance = new CalibrationDataGenerator.CalibrationInstance();
                        instance.vector = testRegData.getRow(i);
                        instance.correctness=testRegData.getLabels()[i];
                        return instance;
                    })
                    .collect(Collectors.toList());
            eval(instances, setCalibrator, logger);
        }



        double[] calibratedProbs = IntStream.range(0,testRegData.getNumDataPoints()).parallel()
                .mapToDouble(i->setCalibrator.calibrate(testRegData.getRow(i))).toArray();
        String result = PrintUtil.toMutipleLines(calibratedProbs);
        FileUtils.writeStringToFile(Paths.get(config.getString("output.dir"),"test_calibrated_probabilities.txt").toFile(),result);
        logger.info("calibrated probabilities for test predictions are saved in "+Paths.get(config.getString("output.dir"),"test_calibrated_probabilities.txt").toFile().getAbsolutePath());

        Map<Integer,Res> bestPredictions = new HashMap<>();
        for (int i=0;i<ids.length;i++){
            int id = ids[i];
            MultiLabel prediction = predictions[i];
            double confidence = calibratedProbs[i];
            Res res = new Res(prediction,confidence);
            if (bestPredictions.containsKey(id)){
                Res old = bestPredictions.get(id);
                if (res.confidence>old.confidence){
                    bestPredictions.put(id,res);
                }
            } else {
                bestPredictions.put(id,res);
            }
        }

        MultiLabel[] rerankedPredictions = new MultiLabel[test.getNumDataPoints()];
        for (int i=0;i<rerankedPredictions.length;i++){
            rerankedPredictions[i] = bestPredictions.get(i).prediction;
        }

        StringBuilder stringBuilder = new StringBuilder();
        for (int i=0;i<rerankedPredictions.length;i++){
            stringBuilder.append(i).append(": ").append(rerankedPredictions[i].toSimpleString()).append("\n");
        }
        FileUtils.writeStringToFile(Paths.get(config.getString("output.dir"),"test_reranked_predictions.txt").toFile(),stringBuilder.toString());

        logger.info("test performance");
        logger.info(new MLMeasures(test.getNumClasses(),test.getMultiLabels(),rerankedPredictions).toString());
    }


    private static BRCalibration.CaliRes eval(List<CalibrationDataGenerator.CalibrationInstance> predictions, VectorCalibrator calibrator, Logger logger){
        double mse = CalibrationEval.mse(generateStream(predictions,calibrator));
        double ace = CalibrationEval.absoluteError(generateStream(predictions,calibrator),10);
        double sharpness = CalibrationEval.sharpness(generateStream(predictions,calibrator),10);
        logger.info("mse="+mse);
        logger.info("absolute calibration error="+ace);
        logger.info("square calibration error="+CalibrationEval.squareError(generateStream(predictions,calibrator),10));
        logger.info("sharpness="+sharpness);
        logger.info("uncertainty="+CalibrationEval.variance(generateStream(predictions,calibrator)));
        logger.info(Displayer.displayCalibrationResult(generateStream(predictions,calibrator)));
        BRCalibration.CaliRes caliRes = new BRCalibration.CaliRes();
        caliRes.mse = mse;
        caliRes.ace= ace;
        caliRes.sharpness = sharpness;
        return caliRes;
    }



    private static Stream<Pair<Double,Double>> generateStream(List<CalibrationDataGenerator.CalibrationInstance> predictions, VectorCalibrator vectorCalibrator){
        return predictions.stream()
                .parallel().map(pred->new Pair<>(vectorCalibrator.calibrate(pred.vector),pred.correctness));
    }


    private static class Res{
        public Res(MultiLabel prediction, double confidence) {
            this.prediction = prediction;
            this.confidence = confidence;
        }

        MultiLabel prediction;
        double confidence;
    }

}

<MSG> calibration

<DFF> @@ -6,6 +6,7 @@ import edu.neu.ccs.pyramid.eval.SafeDivide;
 import edu.neu.ccs.pyramid.multilabel_classification.PluginPredictor;
 import edu.neu.ccs.pyramid.multilabel_classification.imlgb.*;
 import edu.neu.ccs.pyramid.multilabel_classification.thresholding.TunedMarginalClassifier;
+import edu.neu.ccs.pyramid.regression.IsotonicRegression;
 import edu.neu.ccs.pyramid.util.Serialization;
 import org.apache.commons.io.FileUtils;
 import org.apache.mahout.math.Vector;
@@ -23,17 +24,42 @@ public class Calibration {
         }
 
         Config config = new Config(args[0]);
-        System.out.println("=========original probabilities==============");
-        original(config);
-        calibration(config);
+        MultiLabelClfDataSet validation = TRECFormat.loadMultiLabelClfDataSet(config.getString("input.validationSet"),DataSetType.ML_CLF_SPARSE,true);
+        MultiLabelClfDataSet test = TRECFormat.loadMultiLabelClfDataSet(config.getString("input.testSet"),DataSetType.ML_CLF_SPARSE,true);
+        IMLGradientBoosting boosting = (IMLGradientBoosting)Serialization.deserialize(config.getString("input.model"));
+        IMLGBPlattScaling plattScaling = new IMLGBPlattScaling(boosting, validation);
+        IMLGBIsotonicScaling isotonicScaling = new IMLGBIsotonicScaling(boosting, validation);
+
+        System.out.println("====original probabilities on the validation set ====");
+        
+        original(config, boosting, validation);
+
+        System.out.println("====Platt scaling calibrated probabilities on the validation set ====");
+
+        plattCalibration(config, boosting, validation, plattScaling);
+
+        System.out.println("====Isotonic regression calibrated probabilities on the validation set ====");
+
+        isoCalibration(config, boosting, validation, isotonicScaling);
+
+
+        System.out.println("====original probabilities on the test set ====");
+
+        original(config, boosting, test);
+
+        System.out.println("====Platt scaling calibrated probabilities on the test set ====");
+
+        plattCalibration(config, boosting, test, plattScaling);
+
+        System.out.println("====Isotonic regression calibrated probabilities on the test set ====");
+
+        isoCalibration(config, boosting, test, isotonicScaling);
 
     }
 
-    private static void original(Config config) throws Exception{
-        IMLGradientBoosting boosting = (IMLGradientBoosting)Serialization.deserialize(config.getString("input.model"));
+    private static void original(Config config, IMLGradientBoosting boosting, MultiLabelClfDataSet dataSet) throws Exception{
+
         File modelFolder = (new File(config.getString("input.model"))).getParentFile();
-        MultiLabelClfDataSet dataSet = TRECFormat.loadMultiLabelClfDataSet(config.getString("input.data"), DataSetType.ML_CLF_SPARSE,
-                true);
         int numIntervals = 10;
         String predictTarget = config.getString("predict.Target");
         PluginPredictor<IMLGradientBoosting> pluginPredictorTmp = null;
@@ -101,18 +127,12 @@ public class Calibration {
 
     }
 
-    private static void calibration(Config config) throws Exception{
-        IMLGradientBoosting boosting = (IMLGradientBoosting)Serialization.deserialize(config.getString("input.model"));
-        File modelFolder = (new File(config.getString("input.model"))).getParentFile();
-        MultiLabelClfDataSet dataSet = TRECFormat.loadMultiLabelClfDataSet(config.getString("input.data"), DataSetType.ML_CLF_SPARSE,
-                true);
-
-        IMLGBPlattScaling scaling = new IMLGBPlattScaling(boosting, dataSet);
 
+    private static void plattCalibration(Config config, IMLGradientBoosting boosting, MultiLabelClfDataSet dataSet, IMLGBPlattScaling scaling) throws Exception{
         int numIntervals = 10;
         String predictTarget = config.getString("predict.Target");
         PluginPredictor<IMLGradientBoosting> pluginPredictorTmp = null;
-
+        File modelFolder = (new File(config.getString("input.model"))).getParentFile();
         switch (predictTarget){
             case "subsetAccuracy":
                 pluginPredictorTmp = new SubsetAccPredictor(boosting);
@@ -147,7 +167,6 @@ public class Calibration {
 
         double intervalSize = 1.0/numIntervals;
         DecimalFormat decimalFormat = new DecimalFormat("#0.00");
-        System.out.println("=========calibrated probabilities==============");
         System.out.println("interval"+"\t"+"total"+"\t"+"correct"+"\t\t"+"incorrect"+"\t"+"accuracy"+"\t"+"average confidence");
         for (int i=0;i<numIntervals;i++){
             double left = intervalSize*i;
@@ -160,25 +179,83 @@ public class Calibration {
             System.out.println("["+decimalFormat.format(left)+", "+decimalFormat.format(right)+"]"+"\t"+matched.size()+"\t"+numPos+"\t\t"+numNeg+"\t\t"+decimalFormat.format(accuracy)+"\t\t"+decimalFormat.format(aveProb));
         }
 
-        dump(0.7,0.8, dataSet, config, results);
-        dump(0.8,0.9, dataSet, config, results);
-        dump(0.9,1, dataSet, config, results);
+//        dump(0.7,0.8, dataSet, config, results);
+//        dump(0.8,0.9, dataSet, config, results);
+//        dump(0.9,1, dataSet, config, results);
 
     }
 
-    private static void dump(double left, double right, MultiLabelClfDataSet dataSet, Config config, List<Result> results) throws Exception{
-        File data = new File(config.getString("input.data")+"_calibration_"+left+"-"+right);
-        List<Integer>ids = results.stream().filter(res->res.probability>=left&&res.probability<=right).map(res->res.intId).collect(Collectors.toList());
-        if (ids.size()>0){
-            MultiLabelClfDataSet subset = DataSetUtil.sampleData(dataSet,ids);
-            TRECFormat.save(subset, data);
-            //todo this is a hack
-            FileUtils.copyFile(new File(config.getString("input.data"),"data_config.json"), new File(data, "data_config.json"));
-            System.out.println("saved data with calibrated probabilities between "+left+" and "+right+" to "+data.getAbsolutePath());
+
+    private static void isoCalibration(Config config, IMLGradientBoosting boosting, MultiLabelClfDataSet dataSet, IMLGBIsotonicScaling scaling) throws Exception{
+        int numIntervals = 10;
+        String predictTarget = config.getString("predict.Target");
+        PluginPredictor<IMLGradientBoosting> pluginPredictorTmp = null;
+        File modelFolder = (new File(config.getString("input.model"))).getParentFile();
+        switch (predictTarget){
+            case "subsetAccuracy":
+                pluginPredictorTmp = new SubsetAccPredictor(boosting);
+                break;
+            case "hammingLoss":
+                pluginPredictorTmp = new HammingPredictor(boosting);
+                break;
+            case "instanceFMeasure":
+                pluginPredictorTmp = new InstanceF1Predictor(boosting);
+                break;
+            case "macroFMeasure":
+                TunedMarginalClassifier tunedMarginalClassifier = (TunedMarginalClassifier)Serialization.deserialize(new File(modelFolder, "predictor_macro_f"));
+                pluginPredictorTmp = new MacroF1Predictor(boosting,tunedMarginalClassifier);
+                break;
+            default:
+                throw new IllegalArgumentException("unknown prediction target measure "+predictTarget);
         }
 
+        final  PluginPredictor<IMLGradientBoosting> pluginPredictor = pluginPredictorTmp;
+        List<Result> results = IntStream.range(0, dataSet.getNumDataPoints()).parallel()
+                .mapToObj(i->{
+                    Result result = new Result();
+                    Vector vector = dataSet.getRow(i);
+                    MultiLabel multiLabel = pluginPredictor.predict(vector);
+
+                    double probability = scaling.calibratedProb(dataSet.getRow(i),multiLabel);
+                    result.probability = probability;
+                    result.correctness = multiLabel.equals(dataSet.getMultiLabels()[i]);
+                    result.intId =i;
+                    return result;
+                }).collect(Collectors.toList());
+
+        double intervalSize = 1.0/numIntervals;
+        DecimalFormat decimalFormat = new DecimalFormat("#0.00");
+        System.out.println("interval"+"\t"+"total"+"\t"+"correct"+"\t\t"+"incorrect"+"\t"+"accuracy"+"\t"+"average confidence");
+        for (int i=0;i<numIntervals;i++){
+            double left = intervalSize*i;
+            double right = intervalSize*(i+1);
+            List<Result> matched = results.stream().filter(result -> (result.probability>=left && result.probability<=right)).collect(Collectors.toList());
+            int numPos = (int)matched.stream().filter(res->res.correctness).count();
+            int numNeg = matched.size()-numPos;
+            double aveProb = matched.stream().mapToDouble(res->res.probability).average().orElse(0);
+            double accuracy = SafeDivide.divide(numPos,matched.size(), 0);
+            System.out.println("["+decimalFormat.format(left)+", "+decimalFormat.format(right)+"]"+"\t"+matched.size()+"\t"+numPos+"\t\t"+numNeg+"\t\t"+decimalFormat.format(accuracy)+"\t\t"+decimalFormat.format(aveProb));
+        }
+
+//        dump(0.7,0.8, dataSet, config, results);
+//        dump(0.8,0.9, dataSet, config, results);
+//        dump(0.9,1, dataSet, config, results);
+
     }
 
+//    private static void dump(double left, double right, MultiLabelClfDataSet dataSet, Config config, List<Result> results) throws Exception{
+//        File data = new File(config.getString("input.data")+"_calibration_"+left+"-"+right);
+//        List<Integer>ids = results.stream().filter(res->res.probability>=left&&res.probability<=right).map(res->res.intId).collect(Collectors.toList());
+//        if (ids.size()>0){
+//            MultiLabelClfDataSet subset = DataSetUtil.sampleData(dataSet,ids);
+//            TRECFormat.save(subset, data);
+//            //todo this is a hack
+//            FileUtils.copyFile(new File(config.getString("input.data"),"data_config.json"), new File(data, "data_config.json"));
+//            System.out.println("saved data with calibrated probabilities between "+left+" and "+right+" to "+data.getAbsolutePath());
+//        }
+//
+//    }
+
     static class Result{
         int intId;
         double probability;

calibration