<NME> KLLogisticLoss.java
<BEF> ADDFILE
<MSG> KLLogisticLoss

<DFF> @@ -0,0 +1,208 @@
+package edu.neu.ccs.pyramid.classification.logistic_regression;
+
+import edu.neu.ccs.pyramid.dataset.ClfDataSet;
+import edu.neu.ccs.pyramid.dataset.DataSet;
+import edu.neu.ccs.pyramid.dataset.GradientMatrix;
+import edu.neu.ccs.pyramid.dataset.ProbabilityMatrix;
+import edu.neu.ccs.pyramid.optimization.Optimizable;
+import org.apache.mahout.math.DenseVector;
+import org.apache.mahout.math.Vector;
+
+import java.util.stream.IntStream;
+
+/**
+ * Created by chengli on 10/7/15.
+ */
+public class KLLogisticLoss implements Optimizable.ByGradientValue{
+    private LogisticRegression logisticRegression;
+    private DataSet dataSet;
+    private double gaussianPriorVariance;
+    private Vector empiricalCounts;
+    private Vector predictedCounts;
+    private Vector gradient;
+    private int numParameters;
+    private int numClasses;
+    /**
+     * numDataPoints by numClasses;
+     */
+    private ProbabilityMatrix probabilityMatrix;
+
+    //todo the concept is not unified in logistic regression and gradient boosting
+
+    /**
+     * actually negative gradient
+     *  y_ik - p_k(x_i)
+     * numClasses by numDataPoints
+     */
+    private GradientMatrix gradientMatrix;
+    private double value;
+    private boolean isGradientCacheValid;
+    private boolean isValueCacheValid;
+
+    /**
+     *
+     * @param logisticRegression
+     * @param dataSet
+     * @param targetDistribution format: [data point][class]
+     * @param gaussianPriorVariance
+     */
+    public KLLogisticLoss(LogisticRegression logisticRegression,
+                        DataSet dataSet, double[][] targetDistribution,
+                        double gaussianPriorVariance) {
+        this.logisticRegression = logisticRegression;
+        numParameters = logisticRegression.getWeights().totalSize();
+        this.dataSet = dataSet;
+        this.gaussianPriorVariance = gaussianPriorVariance;
+        this.empiricalCounts = new DenseVector(numParameters);
+        this.predictedCounts = new DenseVector(numParameters);
+        this.numClasses = targetDistribution[0].length;
+        this.probabilityMatrix = new ProbabilityMatrix(dataSet.getNumDataPoints(),numClasses);
+        this.gradientMatrix = new GradientMatrix(dataSet.getNumDataPoints(),numClasses, GradientMatrix.Objective.MAXIMIZE);
+        this.updateEmpricalCounts();
+        this.isValueCacheValid=false;
+        this.isGradientCacheValid=false;
+    }
+
+
+
+    public Vector getParameters(){
+        return logisticRegression.getWeights().getAllWeights();
+    }
+
+
+
+    @Override
+    public void setParameters(Vector parameters) {
+        this.logisticRegression.getWeights().setWeightVector(parameters);
+        this.isValueCacheValid=false;
+        this.isGradientCacheValid=false;
+
+    }
+
+
+    public double getValue() {
+        if (isValueCacheValid){
+            return this.value;
+        }
+        Vector parameters = getParameters();
+        this.value =  -1*logisticRegression.dataSetLogLikelihood(dataSet) + parameters.dot(parameters)/(2*gaussianPriorVariance);
+        this.isValueCacheValid = true;
+        return this.value;
+    }
+
+
+
+    public Vector getGradient(){
+        if (isGradientCacheValid){
+            return this.gradient;
+        }
+        updateClassProbMatrix();
+        updatePredictedCounts();
+        updateGradient();
+        this.isGradientCacheValid = true;
+        return this.gradient;
+    }
+
+
+    private void updateGradient(){
+        Vector weights = this.logisticRegression.getWeights().getAllWeights();
+        this.gradient = this.predictedCounts.minus(empiricalCounts).plus(weights.divide(gaussianPriorVariance));
+    }
+
+    private void updateEmpricalCounts(){
+        IntStream.range(0, numParameters).parallel()
+                .forEach(i -> this.empiricalCounts.set(i, calEmpricalCount(i)));
+    }
+
+    private void updatePredictedCounts(){
+        IntStream.range(0,numParameters).parallel()
+                .forEach(i -> this.predictedCounts.set(i, calPredictedCount(i)));
+    }
+
+    private double calEmpricalCount(int parameterIndex){
+        int classIndex = logisticRegression.getWeights().getClassIndex(parameterIndex);
+        int[] labels = dataSet.getLabels();
+        int featureIndex = logisticRegression.getWeights().getFeatureIndex(parameterIndex);
+        double count = 0;
+        //bias
+        if (featureIndex == -1){
+            for (int i=0;i<dataSet.getNumDataPoints();i++){
+                if (labels[i]==classIndex){
+                    count +=1;
+                }
+            }
+        } else {
+            Vector featureColumn = dataSet.getColumn(featureIndex);
+            for (Vector.Element element: featureColumn.nonZeroes()){
+                int dataPointIndex = element.index();
+                double featureValue = element.get();
+                int label = labels[dataPointIndex];
+                if (label==classIndex){
+                    count += featureValue;
+                }
+            }
+        }
+        return count;
+    }
+
+    private double calPredictedCount(int parameterIndex){
+        int classIndex = logisticRegression.getWeights().getClassIndex(parameterIndex);
+        int featureIndex = logisticRegression.getWeights().getFeatureIndex(parameterIndex);
+        double count = 0;
+        double[] probs = this.probabilityMatrix.getProbabilitiesForClass(classIndex);
+        //bias
+        if (featureIndex == -1){
+            for (int i=0;i<dataSet.getNumDataPoints();i++){
+                count += probs[i];
+            }
+        } else {
+            Vector featureColumn = dataSet.getColumn(featureIndex);
+            for (Vector.Element element: featureColumn.nonZeroes()){
+                int dataPointIndex = element.index();
+                double featureValue = element.get();
+                count += probs[dataPointIndex] * featureValue;
+            }
+        }
+        return count;
+    }
+
+    private void updateClassProbs(int dataPointIndex){
+        double[] probs = logisticRegression.predictClassProbs(dataSet.getRow(dataPointIndex));
+        for (int k=0;k<numClasses;k++){
+            this.probabilityMatrix.setProbability(dataPointIndex,k,probs[k]);
+        }
+    }
+
+    private void updateClassProbMatrix(){
+        IntStream.range(0,dataSet.getNumDataPoints()).parallel()
+                .forEach(this::updateClassProbs);
+    }
+
+    private void updataDataGradient(int dataPointIndex){
+        double[] classProbs = this.probabilityMatrix.getProbabilitiesForData(dataPointIndex);
+        int label = dataSet.getLabels()[dataPointIndex];
+        for (int k=0;k<numClasses;k++){
+            if (k==label){
+                this.gradientMatrix.setGradient(dataPointIndex,k,1 - classProbs[k]);
+            } else {
+                this.gradientMatrix.setGradient(dataPointIndex,k,0 - classProbs[k]);
+            }
+        }
+    }
+
+    private void updateDataGradientMatrix(){
+        IntStream.range(0,dataSet.getNumDataPoints()).parallel()
+                .forEach(this::updataDataGradient);
+    }
+
+
+    public ProbabilityMatrix getProbabilityMatrix() {
+        return probabilityMatrix;
+    }
+
+    public GradientMatrix getGradientMatrix() {
+        return gradientMatrix;
+    }
+
+
+}

KLLogisticLoss