<NME> IMLGBInspector.java
<BEF> package edu.neu.ccs.pyramid.multilabel_classification.imlgb;

import edu.neu.ccs.pyramid.dataset.*;
import edu.neu.ccs.pyramid.feature.Feature;
import edu.neu.ccs.pyramid.feature.TopFeatures;
import edu.neu.ccs.pyramid.multilabel_classification.DynamicProgramming;
import edu.neu.ccs.pyramid.multilabel_classification.MultiLabelPredictionAnalysis;
import edu.neu.ccs.pyramid.multilabel_classification.PluginPredictor;
import edu.neu.ccs.pyramid.multilabel_classification.br.SupportPredictor;
import edu.neu.ccs.pyramid.regression.*;
import edu.neu.ccs.pyramid.regression.regression_tree.TreeRule;
import edu.neu.ccs.pyramid.regression.regression_tree.RegTreeInspector;
import edu.neu.ccs.pyramid.regression.regression_tree.RegressionTree;
import edu.neu.ccs.pyramid.util.Pair;
import org.apache.mahout.math.Vector;

import java.util.*;
import java.util.stream.Collectors;
import java.util.stream.IntStream;

/**
 * Created by chengli on 10/13/14.
 */
public class IMLGBInspector {

    /**
     *
     * @param boosting
     * @param classIndex
     * @return features selected by the model
     */
    public static Set<Integer> getSelectedFeatures(IMLGradientBoosting boosting, int classIndex){
        List<Regressor> regressors = boosting.getRegressors(classIndex);
        List<RegressionTree> trees = regressors.stream().filter(regressor ->
                regressor instanceof RegressionTree)
                .map(regressor -> (RegressionTree) regressor)
                .collect(Collectors.toList());
        Set<Integer> features = new HashSet<>();
        for (RegressionTree tree: trees){
            features.addAll(RegTreeInspector.features(tree));
        }
        return features;
    }


    public static Set<Integer> getSelectedFeatures(IMLGradientBoosting boosting){

        Set<Integer> features = new HashSet<>();
        for (int i=0;i<boosting.getNumClasses();i++){
            features.addAll(getSelectedFeatures(boosting, i));
        }
        return features;
    }


    //todo: consider newton step and learning rate

    /**
     * only trees are considered
     * @param boosting
     * @param classIndex
     * @return list of feature index and feature name pairs
     */
    public static TopFeatures topFeatures(IMLGradientBoosting boosting, int classIndex, int limit){
        Map<Feature,Double> totalContributions = new HashMap<>();
        List<Regressor> regressors = boosting.getRegressors(classIndex);
        List<RegressionTree> trees = regressors.stream().filter(regressor ->
                regressor instanceof RegressionTree)
                .map(regressor -> (RegressionTree) regressor)
                .collect(Collectors.toList());
        for (RegressionTree tree: trees){
            Map<Feature,Double> contributions = RegTreeInspector.featureImportance(tree);
            for (Map.Entry<Feature,Double> entry: contributions.entrySet()){
                Feature feature = entry.getKey();
                Double contribution = entry.getValue();
                double oldValue = totalContributions.getOrDefault(feature,0.0);
                double newValue = oldValue+contribution;
                totalContributions.put(feature,newValue);
            }
        }
        Comparator<Map.Entry<Feature,Double>> comparator = Comparator.comparing(Map.Entry::getValue);
        List<Map.Entry<Feature,Double>> topKList = totalContributions.entrySet().stream().sorted(comparator.reversed()).limit(limit).collect(Collectors.toList());
        List<Feature> list = topKList.stream().map(Map.Entry::getKey).collect(Collectors.toList());
        List<Double> utilities = topKList.stream().map(Map.Entry::getValue).collect(Collectors.toList());
        TopFeatures topFeatures = new TopFeatures();
        topFeatures.setUtilities(utilities);
        topFeatures.setTopFeatures(list);
        topFeatures.setClassIndex(classIndex);
        LabelTranslator labelTranslator = boosting.getLabelTranslator();
        topFeatures.setClassName(labelTranslator.toExtLabel(classIndex));
        return topFeatures;
    }


//    public static TopFeatures topFeatures(IMLGradientBoosting boosting,  int classIndex, int limit,Collection<FeatureDistribution> inputDistributions){
//        Map<Feature,Double> totalContributions = new HashMap<>();
//        List<Regressor> regressors = boosting.getRegressors(classIndex);
//        List<RegressionTree> trees = regressors.stream().filter(regressor ->
//                regressor instanceof RegressionTree)
//                .map(regressor -> (RegressionTree) regressor)
//                .collect(Collectors.toList());
//        for (RegressionTree tree: trees){
//            Map<Feature,Double> contributions = RegTreeInspector.featureImportance(tree);
//            for (Map.Entry<Feature,Double> entry: contributions.entrySet()){
//                Feature feature = entry.getKey();
//                Double contribution = entry.getValue();
//                double oldValue = totalContributions.getOrDefault(feature,0.0);
//                double newValue = oldValue+contribution;
//                totalContributions.put(feature,newValue);
//            }
//        }
//        Comparator<Map.Entry<Feature,Double>> comparator = Comparator.comparing(Map.Entry::getValue);
//        List<Feature> list = totalContributions.entrySet().stream().sorted(comparator.reversed()).limit(limit)
//                .map(Map.Entry::getKey).collect(Collectors.toList());
//        TopFeatures topFeatures = new TopFeatures();
//        topFeatures.setTopFeatures(list);
//        topFeatures.setClassIndex(classIndex);
//        LabelTranslator labelTranslator = boosting.getLabelTranslator();
//        topFeatures.setClassName(labelTranslator.toExtLabel(classIndex));
//
//        List<FeatureDistribution> featureDistributions = new ArrayList<>();
//
//
////        for (Feature feature: list){
////            feature.clearIndex();
    }


    public static ClassScoreCalculation decisionProcess(IMLGradientBoosting boosting, LabelTranslator labelTranslator,
                                                        Vector vector, int classIndex, int limit){
        ClassScoreCalculation classScoreCalculation = new ClassScoreCalculation(classIndex,labelTranslator.toExtLabel(classIndex),
                boosting.predictClassScore(vector,classIndex));
        double prob = boosting.predictClassProb(vector,classIndex);
        classScoreCalculation.setClassProbability(prob);
        List<Regressor> regressors = boosting.getRegressors(classIndex);
        List<TreeRule> treeRules = new ArrayList<>();
//                    }
//                }
//                featureDistributions.add(featureDistribution);
//            }
//        }
//
//        topFeatures.setFeatureDistributions(featureDistributions);
//        return topFeatures;
//    }


    public static Map<List<Integer>, Double> countPathMatches(IMLGradientBoosting boosting, DataSet dataSet, int classIndex){
        List<RegressionTree> trees = boosting.getRegressors(classIndex)
                .stream().filter(regressor ->
                regressor instanceof RegressionTree)
                .map(regressor -> (RegressionTree) regressor)
                .collect(Collectors.toList());
        Map<List<Integer>, Double> map = new HashMap<>();
        for (int i=0;i<dataSet.getNumDataPoints();i++){
            List<Integer> path = RegTreeInspector.getMatchedPath(trees,dataSet.getRow(i));
            double oldCount = map.getOrDefault(path,0.0);
            map.put(path,oldCount+1);
        }
        return map;
    }


    public static ClassScoreCalculation decisionProcess(IMLGradientBoosting boosting, LabelTranslator labelTranslator, double prob,
                                                        Vector vector, int classIndex, int limit){
        ClassScoreCalculation classScoreCalculation = new ClassScoreCalculation(classIndex,labelTranslator.toExtLabel(classIndex),
                boosting.predictClassScore(vector,classIndex));
        classScoreCalculation.setClassProbability(prob);
        List<Regressor> regressors = boosting.getRegressors(classIndex);
        List<TreeRule> treeRules = new ArrayList<>();
        for (Regressor regressor : regressors) {
            if (regressor instanceof ConstantRegressor) {
                Rule rule = new ConstantRule(((ConstantRegressor) regressor).getScore());
                classScoreCalculation.addRule(rule);
            }

            if (regressor instanceof RegressionTree) {
                RegressionTree tree = (RegressionTree) regressor;
                TreeRule treeRule = new TreeRule(tree, vector);
                treeRules.add(treeRule);
            }
        }
        Comparator<TreeRule> comparator = Comparator.comparing(decision -> Math.abs(decision.getScore()));
        List<TreeRule> merged = TreeRule.merge(treeRules).stream().sorted(comparator.reversed())
                .limit(limit).collect(Collectors.toList());
        for (TreeRule treeRule : merged){
            classScoreCalculation.addRule(treeRule);
        }

        return classScoreCalculation;
    }

    public static List<LabelModel> getAllRules(IMLGradientBoosting boosting){
        return IntStream.range(0, boosting.getNumClasses()).parallel()
                .mapToObj(k->new LabelModel(boosting, k)).collect(Collectors.toList());

    }







}

        List<ClassScoreCalculation> classScoreCalculations = new ArrayList<>();
        for (int k: classes){
            ClassScoreCalculation classScoreCalculation = decisionProcess(boosting,labelTranslator,
                    dataSet.getRow(dataPointIndex),k,ruleLimit);
            classScoreCalculations.add(classScoreCalculation);
        }
                    MultiLabelPredictionAnalysis.ClassRankInfo rankInfo = new MultiLabelPredictionAnalysis.ClassRankInfo();
                    rankInfo.setClassIndex(label);
                    rankInfo.setClassName(labelTranslator.toExtLabel(label));
                    rankInfo.setProb(boosting.predictClassProb(dataSet.getRow(dataPointIndex), label));
                    return rankInfo;
                }
            ).collect(Collectors.toList());


    public static  MultiLabelPredictionAnalysis analyzePredictionCalibrated(IMLGradientBoosting boosting,
                                                                  IMLGBIsotonicScaling scaling,
                                                                  PluginPredictor<IMLGradientBoosting> pluginPredictor,
                                                                  MultiLabelClfDataSet dataSet,
                                                                  int dataPointIndex,  int ruleLimit,
        List<String> labels = dataSet.getMultiLabels()[dataPointIndex].getMatchedLabelsOrdered().stream()
                .map(labelTranslator::toExtLabel).collect(Collectors.toList());
        predictionAnalysis.setLabels(labels);
        predictionAnalysis.setProbForTrueLabels(scaling.calibratedProb(dataSet.getRow(dataPointIndex),dataSet.getMultiLabels()[dataPointIndex]));


        MultiLabel predictedLabels = pluginPredictor.predict(dataSet.getRow(dataPointIndex));
        List<String> prediction = internalPrediction.stream().map(labelTranslator::toExtLabel).collect(Collectors.toList());
        predictionAnalysis.setPrediction(prediction);

        predictionAnalysis.setProbForPredictedLabels(scaling.calibratedProb(dataSet.getRow(dataPointIndex),predictedLabels));

        double[] classProbs = boosting.predictClassProbs(dataSet.getRow(dataPointIndex));
        List<Integer> classes = new ArrayList<Integer>();
        for (int k = 0; k < boosting.getNumClasses(); k++){
            if (classProbs[k]>=classProbThreshold
                    ||dataSet.getMultiLabels()[dataPointIndex].matchClass(k)
                    ||predictedLabels.matchClass(k)){
                classes.add(k);

        List<ClassScoreCalculation> classScoreCalculations = new ArrayList<>();
        for (int k: classes){
            ClassScoreCalculation classScoreCalculation = decisionProcess(boosting,labelTranslator,
                    dataSet.getRow(dataPointIndex),k,ruleLimit);
            classScoreCalculations.add(classScoreCalculation);
        }
                    MultiLabelPredictionAnalysis.ClassRankInfo rankInfo = new MultiLabelPredictionAnalysis.ClassRankInfo();
                    rankInfo.setClassIndex(label);
                    rankInfo.setClassName(labelTranslator.toExtLabel(label));
                    rankInfo.setProb(boosting.predictClassProb(dataSet.getRow(dataPointIndex), label));
                    return rankInfo;
                }
        ).collect(Collectors.toList());

        if (pluginPredictor instanceof SubsetAccPredictor || pluginPredictor instanceof InstanceF1Predictor){
            double[] labelSetProbs = Arrays.stream(boosting.predictAllAssignmentProbsWithConstraint(dataSet.getRow(dataPointIndex)))
                    .map(scaling::calibratedProb).toArray();

            labelSetRanking = IntStream.range(0,boosting.getAssignments().size())
                    .mapToObj(i -> {


    public static  String simplePredictionAnalysisCalibrated(IMLGradientBoosting boosting,
                                                   IMLGBIsotonicScaling scaling,
                                                   PluginPredictor<IMLGradientBoosting> pluginPredictor,
                                                   MultiLabelClfDataSet dataSet,
                                                   int dataPointIndex,  double classProbThreshold){
        String id = dataSet.getIdTranslator().toExtId(dataPointIndex);
        LabelTranslator labelTranslator = dataSet.getLabelTranslator();
        double[] classProbs = boosting.predictClassProbs(dataSet.getRow(dataPointIndex));
        MultiLabel predicted = pluginPredictor.predict(dataSet.getRow(dataPointIndex));

        List<Integer> classes = new ArrayList<Integer>();
        for (int k = 0; k < boosting.getNumClasses(); k++){
            if (classProbs[k]>=classProbThreshold
                    ||dataSet.getMultiLabels()[dataPointIndex].matchClass(k)
                    ||predicted.matchClass(k)){
                classes.add(k);
        }

        Comparator<Pair<Integer,Double>> comparator = Comparator.comparing(pair->pair.getSecond());
        List<Pair<Integer,Double>> list = classes.stream().map(l -> new Pair<Integer, Double>(l, classProbs[l]))
                .sorted(comparator.reversed()).collect(Collectors.toList());
        for (Pair<Integer,Double> pair: list){
            int label = pair.getFirst();
        }


        double probability = scaling.calibratedProb(dataSet.getRow(dataPointIndex),predicted);

        List<Integer> predictedList = predicted.getMatchedLabelsOrdered();
        sb.append(id).append("\t");

<MSG> remove"print" and fixed typo

<DFF> @@ -126,11 +126,10 @@ public class IMLGBInspector {
     }
 
 
-    public static ClassScoreCalculation decisionProcess(IMLGradientBoosting boosting, LabelTranslator labelTranslator,
+    public static ClassScoreCalculation decisionProcess(IMLGradientBoosting boosting, LabelTranslator labelTranslator, double prob,
                                                         Vector vector, int classIndex, int limit){
         ClassScoreCalculation classScoreCalculation = new ClassScoreCalculation(classIndex,labelTranslator.toExtLabel(classIndex),
                 boosting.predictClassScore(vector,classIndex));
-        double prob = boosting.predictClassProb(vector,classIndex);
         classScoreCalculation.setClassProbability(prob);
         List<Regressor> regressors = boosting.getRegressors(classIndex);
         List<TreeRule> treeRules = new ArrayList<>();
@@ -217,7 +216,7 @@ public class IMLGBInspector {
 
         List<ClassScoreCalculation> classScoreCalculations = new ArrayList<>();
         for (int k: classes){
-            ClassScoreCalculation classScoreCalculation = decisionProcess(boosting,labelTranslator,
+            ClassScoreCalculation classScoreCalculation = decisionProcess(boosting,labelTranslator,classProbs[k],
                     dataSet.getRow(dataPointIndex),k,ruleLimit);
             classScoreCalculations.add(classScoreCalculation);
         }
@@ -227,7 +226,7 @@ public class IMLGBInspector {
                     MultiLabelPredictionAnalysis.ClassRankInfo rankInfo = new MultiLabelPredictionAnalysis.ClassRankInfo();
                     rankInfo.setClassIndex(label);
                     rankInfo.setClassName(labelTranslator.toExtLabel(label));
-                    rankInfo.setProb(boosting.predictClassProb(dataSet.getRow(dataPointIndex), label));
+                    rankInfo.setProb(classProbs[label]);
                     return rankInfo;
                 }
             ).collect(Collectors.toList());
@@ -269,7 +268,8 @@ public class IMLGBInspector {
 
 
     public static  MultiLabelPredictionAnalysis analyzePredictionCalibrated(IMLGradientBoosting boosting,
-                                                                  IMLGBIsotonicScaling scaling,
+                                                                  IMLGBIsotonicScaling setScaling,
+                                                                  IMLGBLabelIsotonicScaling labelScaling,
                                                                   PluginPredictor<IMLGradientBoosting> pluginPredictor,
                                                                   MultiLabelClfDataSet dataSet,
                                                                   int dataPointIndex,  int ruleLimit,
@@ -286,7 +286,7 @@ public class IMLGBInspector {
         List<String> labels = dataSet.getMultiLabels()[dataPointIndex].getMatchedLabelsOrdered().stream()
                 .map(labelTranslator::toExtLabel).collect(Collectors.toList());
         predictionAnalysis.setLabels(labels);
-        predictionAnalysis.setProbForTrueLabels(scaling.calibratedProb(dataSet.getRow(dataPointIndex),dataSet.getMultiLabels()[dataPointIndex]));
+        predictionAnalysis.setProbForTrueLabels(setScaling.calibratedProb(dataSet.getRow(dataPointIndex),dataSet.getMultiLabels()[dataPointIndex]));
 
 
         MultiLabel predictedLabels = pluginPredictor.predict(dataSet.getRow(dataPointIndex));
@@ -295,12 +295,13 @@ public class IMLGBInspector {
         List<String> prediction = internalPrediction.stream().map(labelTranslator::toExtLabel).collect(Collectors.toList());
         predictionAnalysis.setPrediction(prediction);
 
-        predictionAnalysis.setProbForPredictedLabels(scaling.calibratedProb(dataSet.getRow(dataPointIndex),predictedLabels));
+        predictionAnalysis.setProbForPredictedLabels(setScaling.calibratedProb(dataSet.getRow(dataPointIndex),predictedLabels));
 
         double[] classProbs = boosting.predictClassProbs(dataSet.getRow(dataPointIndex));
+        double[] calibratedClassProbs = labelScaling.calibratedClassProbs(classProbs);
         List<Integer> classes = new ArrayList<Integer>();
         for (int k = 0; k < boosting.getNumClasses(); k++){
-            if (classProbs[k]>=classProbThreshold
+            if (calibratedClassProbs[k]>=classProbThreshold
                     ||dataSet.getMultiLabels()[dataPointIndex].matchClass(k)
                     ||predictedLabels.matchClass(k)){
                 classes.add(k);
@@ -309,7 +310,7 @@ public class IMLGBInspector {
 
         List<ClassScoreCalculation> classScoreCalculations = new ArrayList<>();
         for (int k: classes){
-            ClassScoreCalculation classScoreCalculation = decisionProcess(boosting,labelTranslator,
+            ClassScoreCalculation classScoreCalculation = decisionProcess(boosting,labelTranslator,calibratedClassProbs[k],
                     dataSet.getRow(dataPointIndex),k,ruleLimit);
             classScoreCalculations.add(classScoreCalculation);
         }
@@ -319,7 +320,7 @@ public class IMLGBInspector {
                     MultiLabelPredictionAnalysis.ClassRankInfo rankInfo = new MultiLabelPredictionAnalysis.ClassRankInfo();
                     rankInfo.setClassIndex(label);
                     rankInfo.setClassName(labelTranslator.toExtLabel(label));
-                    rankInfo.setProb(boosting.predictClassProb(dataSet.getRow(dataPointIndex), label));
+                    rankInfo.setProb(calibratedClassProbs[label]);
                     return rankInfo;
                 }
         ).collect(Collectors.toList());
@@ -330,7 +331,7 @@ public class IMLGBInspector {
 
         if (pluginPredictor instanceof SubsetAccPredictor || pluginPredictor instanceof InstanceF1Predictor){
             double[] labelSetProbs = Arrays.stream(boosting.predictAllAssignmentProbsWithConstraint(dataSet.getRow(dataPointIndex)))
-                    .map(scaling::calibratedProb).toArray();
+                    .map(setScaling::calibratedProb).toArray();
 
             labelSetRanking = IntStream.range(0,boosting.getAssignments().size())
                     .mapToObj(i -> {
@@ -426,7 +427,8 @@ public class IMLGBInspector {
 
 
     public static  String simplePredictionAnalysisCalibrated(IMLGradientBoosting boosting,
-                                                   IMLGBIsotonicScaling scaling,
+                                                   IMLGBIsotonicScaling setScaling,
+                                                   IMLGBLabelIsotonicScaling labelScaling,
                                                    PluginPredictor<IMLGradientBoosting> pluginPredictor,
                                                    MultiLabelClfDataSet dataSet,
                                                    int dataPointIndex,  double classProbThreshold){
@@ -438,11 +440,13 @@ public class IMLGBInspector {
         String id = dataSet.getIdTranslator().toExtId(dataPointIndex);
         LabelTranslator labelTranslator = dataSet.getLabelTranslator();
         double[] classProbs = boosting.predictClassProbs(dataSet.getRow(dataPointIndex));
+        double[] calibratedClassProbs = labelScaling.calibratedClassProbs(classProbs);
+
         MultiLabel predicted = pluginPredictor.predict(dataSet.getRow(dataPointIndex));
 
         List<Integer> classes = new ArrayList<Integer>();
         for (int k = 0; k < boosting.getNumClasses(); k++){
-            if (classProbs[k]>=classProbThreshold
+            if (calibratedClassProbs[k]>=classProbThreshold
                     ||dataSet.getMultiLabels()[dataPointIndex].matchClass(k)
                     ||predicted.matchClass(k)){
                 classes.add(k);
@@ -450,7 +454,7 @@ public class IMLGBInspector {
         }
 
         Comparator<Pair<Integer,Double>> comparator = Comparator.comparing(pair->pair.getSecond());
-        List<Pair<Integer,Double>> list = classes.stream().map(l -> new Pair<Integer, Double>(l, classProbs[l]))
+        List<Pair<Integer,Double>> list = classes.stream().map(l -> new Pair<Integer, Double>(l, calibratedClassProbs[l]))
                 .sorted(comparator.reversed()).collect(Collectors.toList());
         for (Pair<Integer,Double> pair: list){
             int label = pair.getFirst();
@@ -465,7 +469,7 @@ public class IMLGBInspector {
         }
 
 
-        double probability = scaling.calibratedProb(dataSet.getRow(dataPointIndex),predicted);
+        double probability = setScaling.calibratedProb(dataSet.getRow(dataPointIndex),predicted);
 
         List<Integer> predictedList = predicted.getMatchedLabelsOrdered();
         sb.append(id).append("\t");

remove"print" and fixed typo