<NME> Calibration.java
<BEF> package edu.neu.ccs.pyramid.application;

import edu.neu.ccs.pyramid.configuration.Config;
import edu.neu.ccs.pyramid.dataset.*;
import edu.neu.ccs.pyramid.eval.SafeDivide;
import edu.neu.ccs.pyramid.multilabel_classification.PluginPredictor;
import edu.neu.ccs.pyramid.multilabel_classification.imlgb.*;
import edu.neu.ccs.pyramid.util.Serialization;
import org.apache.mahout.math.Vector;

import java.io.File;
import java.text.DecimalFormat;
import java.util.List;
import java.util.logging.Logger;
import java.util.stream.Collectors;
import java.util.stream.IntStream;

public class Calibration {
    public static void main(Config config, Logger logger) throws Exception{
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.logging.FileHandler;
import java.util.logging.Logger;
import java.util.logging.SimpleFormatter;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
import java.util.stream.Stream;

/**

        displaySetCalibration(boosting, test, isotonicScaling, logger);

        labelUncalibration(boosting,test,logger);

        //jointLabelCalibration(boosting, test, valid, logger, config);
            fileHandler.setFormatter(formatter);
            logger.addHandler(fileHandler);
            logger.setUseParentHandlers(false);


        Serialization.serialize(isotonicScaling, new File(config.getString("out"),"set_calibration"));



        if (config.getBoolean("test")){
            test(config, logger);
        }

        if (fileHandler!=null){
            fileHandler.close();
        }
    }
    public static void main(String[] args) throws Exception {
        Config config = new Config(args[0]);
        main(config);

    }


    private static void calibrate(Config config, Logger logger) throws Exception{

        logger.info("start training calibrator");

        MultiLabelClfDataSet train = TRECFormat.loadMultiLabelClfDataSet(config.getString("input.trainData"), DataSetType.ML_CLF_SEQ_SPARSE, true);

        MultiLabelClfDataSet cal = TRECFormat.loadMultiLabelClfDataSet(config.getString("input.calibData"), DataSetType.ML_CLF_SEQ_SPARSE, true);

        MultiLabel[] calPred = loadPredictions(config.getString("input.calibPredictions"));

        int[] calIds = loadIds(config.getString("input.calibPredictions"));
        Vector[] calibScore = loadFeatures(config.getString("input.calibPredictions"));

        Pair<RegDataSet,PredictionFeatureExtractor> pair = createCaliData(cal,calPred,calibScore, calIds, train);
        RegDataSet calibRegData = pair.getFirst();
        PredictionFeatureExtractor predictionFeatureExtractor = pair.getSecond();
        int[] monotonicity = pair.getSecond().featureMonotonicity();

        MultiLabelClfDataSet valid = TRECFormat.loadMultiLabelClfDataSet(config.getString("input.validData"), DataSetType.ML_CLF_SEQ_SPARSE, true);

        MultiLabel[] validPred = loadPredictions(config.getString("input.validPredictions"));

        int[] validIds = loadIds(config.getString("input.validPredictions"));
        Vector[] validScore = loadFeatures(config.getString("input.validPredictions"));

        RegDataSet validRegData = createCaliData(valid,validPred,validScore, validIds, predictionFeatureExtractor).getFirst();

        LSBoost lsBoost = trainCalibrator(calibRegData,validRegData,monotonicity);

        VectorCalibrator vectorCalibrator = new RegressorCalibrator(lsBoost);

    }


    private static void displaySetCalibration(IMLGradientBoosting boosting, MultiLabelClfDataSet dataSet, IMLGBIsotonicScaling scaling, Logger logger) throws Exception{
        int numIntervals = 10;
        PluginPredictor<IMLGradientBoosting> pluginPredictorTmp = new SubsetAccPredictor(boosting);
        final  PluginPredictor<IMLGradientBoosting> pluginPredictor = pluginPredictorTmp;
        List<Result> results = IntStream.range(0, dataSet.getNumDataPoints()).parallel()
                .mapToObj(i->{
                    Result result = new Result();
                    Vector vector = dataSet.getRow(i);
                    MultiLabel multiLabel = pluginPredictor.predict(vector);

                    double probability = scaling.calibratedProb(dataSet.getRow(i),multiLabel);
                    result.probability = probability;
                    result.correctness = multiLabel.equals(dataSet.getMultiLabels()[i]);
                    result.intId =i;
                    return result;
                }).collect(Collectors.toList());

        double intervalSize = 1.0/numIntervals;
        DecimalFormat decimalFormat = new DecimalFormat("#0.00");
        StringBuilder stringBuilder = new StringBuilder();
        stringBuilder.append("calibrated set probability\n");
        stringBuilder.append("\ninterval"+"\t"+"total"+"\t"+"correct"+"\t\t"+"incorrect"+"\t"+"accuracy"+"\t"+"average confidence\n");
        for (int i=0;i<numIntervals;i++){
            double left = intervalSize*i;
            double right = intervalSize*(i+1);
            List<Result> matched = results.stream().filter(result -> (result.probability>=left && result.probability<right)).collect(Collectors.toList());
            if (i==numIntervals-1){
                matched = results.stream().filter(result -> (result.probability>=left && result.probability<=right)).collect(Collectors.toList());
            }
            int numPos = (int)matched.stream().filter(res->res.correctness).count();
            int numNeg = matched.size()-numPos;
            double aveProb = matched.stream().mapToDouble(res->res.probability).average().orElse(0);
            double accuracy = SafeDivide.divide(numPos,matched.size(), 0);
            String st = "["+decimalFormat.format(left)+", "+decimalFormat.format(right)+")"+"\t"+matched.size()+"\t"+numPos+"\t\t"+numNeg+"\t\t"+decimalFormat.format(accuracy)+"\t\t"+decimalFormat.format(aveProb)+"\n";
            if (i==numIntervals-1){
                st = "["+decimalFormat.format(left)+", "+decimalFormat.format(right)+"]"+"\t"+matched.size()+"\t"+numPos+"\t\t"+numNeg+"\t\t"+decimalFormat.format(accuracy)+"\t\t"+decimalFormat.format(aveProb)+"\n";
            }
            stringBuilder.append(st);
        }
        logger.info(stringBuilder.toString());


    }

    private static void jointLabelCalibration(IMLGradientBoosting boosting, MultiLabelClfDataSet testSet, MultiLabelClfDataSet validSet, Logger logger, Config config)throws Exception{
        logger.info("start calibrating joint label probability ");
        IMLGBJointLabelIsotonicScaling imlgbJointLabelIsotonicScaling = new IMLGBJointLabelIsotonicScaling(boosting, validSet);
        logger.info("finish calibrating joint label probability");
        IMLGBJointLabelIsotonicScaling.BucketInfo total = imlgbJointLabelIsotonicScaling.individualProbs(testSet);

        double[] counts = total.getCounts();
        double[] correct = total.getSums();
        double[] sumProbs = total.getSumProbs();
        double[] accs = new double[counts.length];
        double[] average_confidence = new double[counts.length];

        for (int i = 0; i < counts.length; i++) {
            accs[i] = correct[i] / counts[i];
        }
        for (int j = 0; j < counts.length; j++) {
            average_confidence[j] = sumProbs[j] / counts[j];
        }

        DecimalFormat decimalFormat = new DecimalFormat("#0.0000");
        StringBuilder sb = new StringBuilder();
        sb.append("calibrated joint label probabilities\n");
        sb.append("interval\t\t").append("total\t\t").append("correct\t\t").append("incorrect\t\t").append("accuracy\t\t").append("average confidence\n");
        for (int i = 0; i < 10; i++) {
            sb.append("[").append(decimalFormat.format(i * 0.1)).append(",")
                    .append(decimalFormat.format((i + 1) * 0.1)).append("]")
                    .append("\t\t").append(counts[i]).append("\t\t").append(correct[i]).append("\t\t")
                    .append(counts[i] - correct[i]).append("\t\t").append(decimalFormat.format(accs[i])).append("\t\t")
                    .append(decimalFormat.format(average_confidence[i])).append("\n");

        }
        logger.info(sb.toString());
        Serialization.serialize(imlgbJointLabelIsotonicScaling, new File(config.getString("out"),"joint_label_calibration"));

    }

    private static void labelIsoCalibration(IMLGradientBoosting boosting, MultiLabelClfDataSet testSet, MultiLabelClfDataSet validSet, Logger logger, Config config)throws Exception{
        logger.info("start calibrating label probability ");
        IMLGBLabelIsotonicScaling imlgbLabelIsotonicScaling = new IMLGBLabelIsotonicScaling(boosting, validSet);
        PredictionFeatureExtractor predictionFeatureExtractor = new CombinedPredictionFeatureExtractor(extractors);

        for (int i=0;i<predictions.length;i++){
            PredictionCandidate predictionCandidate = new PredictionCandidate();
            predictionCandidate.multiLabel = predictions[i];
            predictionCandidate.x = scores[i];
            Vector row = predictionFeatureExtractor.extractFeatures(predictionCandidate);
            for (int j=0;j<regDataSet.getNumFeatures();j++){
                regDataSet.setFeatureValue(i,j,row.get(j));
            }
            int id = ids[i];
            if (multiLabelClfDataSet.getMultiLabels()[id].equals(predictions[i])){
                regDataSet.setLabel(i,1);
            } else {
                regDataSet.setLabel(i,0);
            }
        }
        return new Pair<>(regDataSet,predictionFeatureExtractor);
    }



    private static Pair<RegDataSet,PredictionFeatureExtractor> createCaliData(MultiLabelClfDataSet multiLabelClfDataSet, MultiLabel[] predictions, Vector[] scores, int[] ids, PredictionFeatureExtractor predictionFeatureExtractor){

        int numFeatures = predictionFeatureExtractor.featureMonotonicity().length;

        RegDataSet regDataSet = RegDataSetBuilder.getBuilder()
                .numDataPoints(predictions.length)
                .numFeatures(numFeatures)
                .build();

        for (int i=0;i<predictions.length;i++){
            PredictionCandidate predictionCandidate = new PredictionCandidate();
            predictionCandidate.multiLabel = predictions[i];
            predictionCandidate.x = scores[i];
            Vector row = predictionFeatureExtractor.extractFeatures(predictionCandidate);
        final int numBuckets = 10;
        double bucketLength = 1.0/numBuckets;

        BucketInfo empty = new BucketInfo(numBuckets);
        BucketInfo total;
        total = IntStream.range(0, dataSet.getNumDataPoints()).parallel()
                .mapToObj(i->{
            } else {
                regDataSet.setLabel(i,0);
            }
        }
        return new Pair<>(regDataSet,predictionFeatureExtractor);
    }


    private static void test(Config config, Logger logger) throws Exception{
        MultiLabelClfDataSet test = TRECFormat.loadMultiLabelClfDataSet(config.getString("input.testData"), DataSetType.ML_CLF_SEQ_SPARSE, true);

        VectorCalibrator setCalibrator = (VectorCalibrator) Serialization.deserialize(Paths.get(config.getString("output.dir"),"set_calibrator").toFile());

        PredictionFeatureExtractor predictionFeatureExtractor = (PredictionFeatureExtractor) Serialization.deserialize(Paths.get(config.getString("output.dir"),"prediction_feature_extractor").toFile());

        MultiLabel[] predictions = loadPredictions(config.getString("input.testPredictions"));
        Vector[] scores = loadFeatures(config.getString("input.testPredictions"));

                        }
                    }
                    return new BucketInfo(count, sum, sumProbs);
                }).reduce(empty, BucketInfo::add, BucketInfo::add);

        double[] counts = total.getCounts();
        double[] correct = total.getSums();
                    .boxed().map(i -> {

                        CalibrationDataGenerator.CalibrationInstance instance = new CalibrationDataGenerator.CalibrationInstance();
                        instance.vector = testRegData.getRow(i);
                        instance.correctness=testRegData.getLabels()[i];
                        return instance;
                    })
                    .collect(Collectors.toList());
            eval(instances, setCalibrator, logger);
        }



        double[] calibratedProbs = IntStream.range(0,testRegData.getNumDataPoints()).parallel()
                .mapToDouble(i->setCalibrator.calibrate(testRegData.getRow(i))).toArray();
        String result = PrintUtil.toMutipleLines(calibratedProbs);
        FileUtils.writeStringToFile(Paths.get(config.getString("output.dir"),"test_calibrated_probabilities.txt").toFile(),result);
        logger.info("calibrated probabilities for test predictions are saved in "+Paths.get(config.getString("output.dir"),"test_calibrated_probabilities.txt").toFile().getAbsolutePath());

        Map<Integer,Res> bestPredictions = new HashMap<>();
        for (int i=0;i<ids.length;i++){
            int id = ids[i];
            MultiLabel prediction = predictions[i];
            double confidence = calibratedProbs[i];
            Res res = new Res(prediction,confidence);
            if (bestPredictions.containsKey(id)){
                Res old = bestPredictions.get(id);
                if (res.confidence>old.confidence){
                    bestPredictions.put(id,res);
                }
            } else {
                bestPredictions.put(id,res);
            }
        }

        MultiLabel[] rerankedPredictions = new MultiLabel[test.getNumDataPoints()];
        for (int i=0;i<rerankedPredictions.length;i++){
            rerankedPredictions[i] = bestPredictions.get(i).prediction;
        }

        StringBuilder stringBuilder = new StringBuilder();
        for (int i=0;i<rerankedPredictions.length;i++){
            stringBuilder.append(i).append(": ").append(rerankedPredictions[i].toSimpleString()).append("\n");
        }
        FileUtils.writeStringToFile(Paths.get(config.getString("output.dir"),"test_reranked_predictions.txt").toFile(),stringBuilder.toString());

        logger.info("test performance");
        logger.info(new MLMeasures(test.getNumClasses(),test.getMultiLabels(),rerankedPredictions).toString());
    }


    private static BRCalibration.CaliRes eval(List<CalibrationDataGenerator.CalibrationInstance> predictions, VectorCalibrator calibrator, Logger logger){
        double mse = CalibrationEval.mse(generateStream(predictions,calibrator));
        double ace = CalibrationEval.absoluteError(generateStream(predictions,calibrator),10);
        double sharpness = CalibrationEval.sharpness(generateStream(predictions,calibrator),10);
        logger.info("mse="+mse);
        logger.info("absolute calibration error="+ace);
        logger.info("square calibration error="+CalibrationEval.squareError(generateStream(predictions,calibrator),10));
        logger.info("sharpness="+sharpness);
        logger.info("uncertainty="+CalibrationEval.variance(generateStream(predictions,calibrator)));
        logger.info(Displayer.displayCalibrationResult(generateStream(predictions,calibrator)));
        BRCalibration.CaliRes caliRes = new BRCalibration.CaliRes();
        caliRes.mse = mse;
        caliRes.ace= ace;
        caliRes.sharpness = sharpness;
        return caliRes;
    }



    private static Stream<Pair<Double,Double>> generateStream(List<CalibrationDataGenerator.CalibrationInstance> predictions, VectorCalibrator vectorCalibrator){
        return predictions.stream()
                .parallel().map(pred->new Pair<>(vectorCalibrator.calibrate(pred.vector),pred.correctness));
    }


    private static class Res{
        public Res(MultiLabel prediction, double confidence) {
            this.prediction = prediction;
            this.confidence = confidence;
        }

        MultiLabel prediction;
        double confidence;
    }

}

<MSG> calibration

<DFF> @@ -1,19 +1,25 @@
 package edu.neu.ccs.pyramid.application;
 
+import com.sun.scenario.effect.impl.sw.sse.SSEBlend_SRC_OUTPeer;
 import edu.neu.ccs.pyramid.configuration.Config;
 import edu.neu.ccs.pyramid.dataset.*;
 import edu.neu.ccs.pyramid.eval.SafeDivide;
 import edu.neu.ccs.pyramid.multilabel_classification.PluginPredictor;
 import edu.neu.ccs.pyramid.multilabel_classification.imlgb.*;
+import edu.neu.ccs.pyramid.regression.IsotonicRegression;
+import edu.neu.ccs.pyramid.util.CalibrationDisplay;
+import edu.neu.ccs.pyramid.util.Pair;
 import edu.neu.ccs.pyramid.util.Serialization;
 import org.apache.mahout.math.Vector;
 
 import java.io.File;
 import java.text.DecimalFormat;
 import java.util.List;
+import java.util.Set;
 import java.util.logging.Logger;
 import java.util.stream.Collectors;
 import java.util.stream.IntStream;
+import java.util.stream.Stream;
 
 public class Calibration {
     public static void main(Config config, Logger logger) throws Exception{
@@ -32,6 +38,13 @@ public class Calibration {
 
         displaySetCalibration(boosting, test, isotonicScaling, logger);
 
+        logger.info("start calibrating set probability by cardinality");
+        CardinalityCalibrator cardinalityCalibrator = new CardinalityCalibrator(boosting, valid);
+        logger.info("finish calibrating set probability by cardinality");
+
+
+        displayCardinalityCalibration(boosting, test, cardinalityCalibrator, logger);
+
         labelUncalibration(boosting,test,logger);
 
         //jointLabelCalibration(boosting, test, valid, logger, config);
@@ -41,6 +54,7 @@ public class Calibration {
 
 
         Serialization.serialize(isotonicScaling, new File(config.getString("out"),"set_calibration"));
+        Serialization.serialize(cardinalityCalibrator, new File(config.getString("out"),"set_calibration_by_cardinality"));
 
 
 
@@ -93,86 +107,102 @@ public class Calibration {
     }
 
 
+//    private static void displaySetCalibration(IMLGradientBoosting boosting, MultiLabelClfDataSet dataSet, IMLGBIsotonicScaling scaling, Logger logger) throws Exception{
+//        int numIntervals = 10;
+//        PluginPredictor<IMLGradientBoosting> pluginPredictorTmp = new SubsetAccPredictor(boosting);
+//        final  PluginPredictor<IMLGradientBoosting> pluginPredictor = pluginPredictorTmp;
+//        List<Result> results = IntStream.range(0, dataSet.getNumDataPoints()).parallel()
+//                .mapToObj(i->{
+//                    Result result = new Result();
+//                    Vector vector = dataSet.getRow(i);
+//                    MultiLabel multiLabel = pluginPredictor.predict(vector);
+//
+//                    double probability = scaling.calibratedProb(dataSet.getRow(i),multiLabel);
+//                    result.probability = probability;
+//                    result.correctness = multiLabel.equals(dataSet.getMultiLabels()[i]);
+//                    result.intId =i;
+//                    return result;
+//                }).collect(Collectors.toList());
+//
+//        double intervalSize = 1.0/numIntervals;
+//        DecimalFormat decimalFormat = new DecimalFormat("#0.00");
+//        StringBuilder stringBuilder = new StringBuilder();
+//        stringBuilder.append("calibrated set probability\n");
+//        stringBuilder.append("\ninterval"+"\t"+"total"+"\t"+"correct"+"\t\t"+"incorrect"+"\t"+"accuracy"+"\t"+"average confidence\n");
+//        for (int i=0;i<numIntervals;i++){
+//            double left = intervalSize*i;
+//            double right = intervalSize*(i+1);
+//            List<Result> matched = results.stream().filter(result -> (result.probability>=left && result.probability<right)).collect(Collectors.toList());
+//            if (i==numIntervals-1){
+//                matched = results.stream().filter(result -> (result.probability>=left && result.probability<=right)).collect(Collectors.toList());
+//            }
+//            int numPos = (int)matched.stream().filter(res->res.correctness).count();
+//            int numNeg = matched.size()-numPos;
+//            double aveProb = matched.stream().mapToDouble(res->res.probability).average().orElse(0);
+//            double accuracy = SafeDivide.divide(numPos,matched.size(), 0);
+//            String st = "["+decimalFormat.format(left)+", "+decimalFormat.format(right)+")"+"\t"+matched.size()+"\t"+numPos+"\t\t"+numNeg+"\t\t"+decimalFormat.format(accuracy)+"\t\t"+decimalFormat.format(aveProb)+"\n";
+//            if (i==numIntervals-1){
+//                st = "["+decimalFormat.format(left)+", "+decimalFormat.format(right)+"]"+"\t"+matched.size()+"\t"+numPos+"\t\t"+numNeg+"\t\t"+decimalFormat.format(accuracy)+"\t\t"+decimalFormat.format(aveProb)+"\n";
+//            }
+//            stringBuilder.append(st);
+//        }
+//        logger.info(stringBuilder.toString());
+//
+//
+//    }
+
+
     private static void displaySetCalibration(IMLGradientBoosting boosting, MultiLabelClfDataSet dataSet, IMLGBIsotonicScaling scaling, Logger logger) throws Exception{
-        int numIntervals = 10;
         PluginPredictor<IMLGradientBoosting> pluginPredictorTmp = new SubsetAccPredictor(boosting);
         final  PluginPredictor<IMLGradientBoosting> pluginPredictor = pluginPredictorTmp;
-        List<Result> results = IntStream.range(0, dataSet.getNumDataPoints()).parallel()
+
+        Stream<Pair<Double,Integer>> stream = IntStream.range(0, dataSet.getNumDataPoints()).parallel()
                 .mapToObj(i->{
-                    Result result = new Result();
+                    Pair<Double,Integer> pairOverall = new Pair<>();
                     Vector vector = dataSet.getRow(i);
                     MultiLabel multiLabel = pluginPredictor.predict(vector);
+                    double prob = boosting.predictAssignmentProbWithConstraint(vector, multiLabel);
+                    pairOverall.setFirst(prob);
+                    pairOverall.setSecond(0);
+                    if (multiLabel.equals(dataSet.getMultiLabels()[i])) {
+                        pairOverall.setSecond(1);
+                    }
+                    return pairOverall;
+                });
 
-                    double probability = scaling.calibratedProb(dataSet.getRow(i),multiLabel);
-                    result.probability = probability;
-                    result.correctness = multiLabel.equals(dataSet.getMultiLabels()[i]);
-                    result.intId =i;
-                    return result;
-                }).collect(Collectors.toList());
-
-        double intervalSize = 1.0/numIntervals;
-        DecimalFormat decimalFormat = new DecimalFormat("#0.00");
-        StringBuilder stringBuilder = new StringBuilder();
-        stringBuilder.append("calibrated set probability\n");
-        stringBuilder.append("\ninterval"+"\t"+"total"+"\t"+"correct"+"\t\t"+"incorrect"+"\t"+"accuracy"+"\t"+"average confidence\n");
-        for (int i=0;i<numIntervals;i++){
-            double left = intervalSize*i;
-            double right = intervalSize*(i+1);
-            List<Result> matched = results.stream().filter(result -> (result.probability>=left && result.probability<right)).collect(Collectors.toList());
-            if (i==numIntervals-1){
-                matched = results.stream().filter(result -> (result.probability>=left && result.probability<=right)).collect(Collectors.toList());
-            }
-            int numPos = (int)matched.stream().filter(res->res.correctness).count();
-            int numNeg = matched.size()-numPos;
-            double aveProb = matched.stream().mapToDouble(res->res.probability).average().orElse(0);
-            double accuracy = SafeDivide.divide(numPos,matched.size(), 0);
-            String st = "["+decimalFormat.format(left)+", "+decimalFormat.format(right)+")"+"\t"+matched.size()+"\t"+numPos+"\t\t"+numNeg+"\t\t"+decimalFormat.format(accuracy)+"\t\t"+decimalFormat.format(aveProb)+"\n";
-            if (i==numIntervals-1){
-                st = "["+decimalFormat.format(left)+", "+decimalFormat.format(right)+"]"+"\t"+matched.size()+"\t"+numPos+"\t\t"+numNeg+"\t\t"+decimalFormat.format(accuracy)+"\t\t"+decimalFormat.format(aveProb)+"\n";
-            }
-            stringBuilder.append(st);
-        }
-        logger.info(stringBuilder.toString());
+        IsotonicRegression isotonicRegression = scaling.getIsotonicRegression();
+        logger.info(isotonicRegression.displayCalibrationResult(stream));
 
 
     }
 
-    private static void jointLabelCalibration(IMLGradientBoosting boosting, MultiLabelClfDataSet testSet, MultiLabelClfDataSet validSet, Logger logger, Config config)throws Exception{
-        logger.info("start calibrating joint label probability ");
-        IMLGBJointLabelIsotonicScaling imlgbJointLabelIsotonicScaling = new IMLGBJointLabelIsotonicScaling(boosting, validSet);
-        logger.info("finish calibrating joint label probability");
-        IMLGBJointLabelIsotonicScaling.BucketInfo total = imlgbJointLabelIsotonicScaling.individualProbs(testSet);
 
-        double[] counts = total.getCounts();
-        double[] correct = total.getSums();
-        double[] sumProbs = total.getSumProbs();
-        double[] accs = new double[counts.length];
-        double[] average_confidence = new double[counts.length];
+    private static void displayCardinalityCalibration(IMLGradientBoosting boosting, MultiLabelClfDataSet dataSet, CardinalityCalibrator scaling, Logger logger) throws Exception{
+        PluginPredictor<IMLGradientBoosting> pluginPredictorTmp = new SubsetAccPredictor(boosting);
+        final  PluginPredictor<IMLGradientBoosting> pluginPredictor = pluginPredictorTmp;
 
-        for (int i = 0; i < counts.length; i++) {
-            accs[i] = correct[i] / counts[i];
-        }
-        for (int j = 0; j < counts.length; j++) {
-            average_confidence[j] = sumProbs[j] / counts[j];
-        }
+        Stream<Pair<Double,Integer>> stream = IntStream.range(0, dataSet.getNumDataPoints()).parallel()
+                .mapToObj(i->{
+                    Pair<Double,Integer> pairOverall = new Pair<>();
+                    Vector vector = dataSet.getRow(i);
+                    MultiLabel multiLabel = pluginPredictor.predict(vector);
+                    double prob = boosting.predictAssignmentProbWithConstraint(vector, multiLabel);
+                    double calibratedProb = scaling.calibrate(prob, multiLabel.getNumMatchedLabels());
+                    pairOverall.setFirst(calibratedProb);
+                    pairOverall.setSecond(0);
+                    if (multiLabel.equals(dataSet.getMultiLabels()[i])) {
+                        pairOverall.setSecond(1);
+                    }
+                    return pairOverall;
+                });
 
-        DecimalFormat decimalFormat = new DecimalFormat("#0.0000");
-        StringBuilder sb = new StringBuilder();
-        sb.append("calibrated joint label probabilities\n");
-        sb.append("interval\t\t").append("total\t\t").append("correct\t\t").append("incorrect\t\t").append("accuracy\t\t").append("average confidence\n");
-        for (int i = 0; i < 10; i++) {
-            sb.append("[").append(decimalFormat.format(i * 0.1)).append(",")
-                    .append(decimalFormat.format((i + 1) * 0.1)).append("]")
-                    .append("\t\t").append(counts[i]).append("\t\t").append(correct[i]).append("\t\t")
-                    .append(counts[i] - correct[i]).append("\t\t").append(decimalFormat.format(accs[i])).append("\t\t")
-                    .append(decimalFormat.format(average_confidence[i])).append("\n");
 
-        }
-        logger.info(sb.toString());
-        Serialization.serialize(imlgbJointLabelIsotonicScaling, new File(config.getString("out"),"joint_label_calibration"));
+        logger.info(CalibrationDisplay.displayCalibrationResult(stream));
+
 
     }
 
+
     private static void labelIsoCalibration(IMLGradientBoosting boosting, MultiLabelClfDataSet testSet, MultiLabelClfDataSet validSet, Logger logger, Config config)throws Exception{
         logger.info("start calibrating label probability ");
         IMLGBLabelIsotonicScaling imlgbLabelIsotonicScaling = new IMLGBLabelIsotonicScaling(boosting, validSet);
@@ -215,7 +245,6 @@ public class Calibration {
         final int numBuckets = 10;
         double bucketLength = 1.0/numBuckets;
 
-        BucketInfo empty = new BucketInfo(numBuckets);
         BucketInfo total;
         total = IntStream.range(0, dataSet.getNumDataPoints()).parallel()
                 .mapToObj(i->{
@@ -240,7 +269,7 @@ public class Calibration {
                         }
                     }
                     return new BucketInfo(count, sum, sumProbs);
-                }).reduce(empty, BucketInfo::add, BucketInfo::add);
+                }).collect(()->new BucketInfo(numBuckets), BucketInfo::addAll, BucketInfo::addAll);
 
         double[] counts = total.getCounts();
         double[] correct = total.getSums();

calibration