<NME> BRLREN.java
<BEF> package edu.neu.ccs.pyramid.application;


import com.fasterxml.jackson.databind.ObjectMapper;
import edu.neu.ccs.pyramid.classification.logistic_regression.LogisticRegression;
import edu.neu.ccs.pyramid.classification.logistic_regression.LogisticRegressionInspector;
import edu.neu.ccs.pyramid.configuration.Config;
import edu.neu.ccs.pyramid.dataset.*;
import edu.neu.ccs.pyramid.eval.MAP;
import edu.neu.ccs.pyramid.eval.MLMeasures;
import edu.neu.ccs.pyramid.feature.Feature;
import edu.neu.ccs.pyramid.feature.FeatureList;
import edu.neu.ccs.pyramid.feature.TopFeatures;
import edu.neu.ccs.pyramid.multilabel_classification.MultiLabelClassifier;
import edu.neu.ccs.pyramid.multilabel_classification.cbm.*;
import edu.neu.ccs.pyramid.multilabel_classification.imlgb.IMLGBInspector;
import edu.neu.ccs.pyramid.optimization.EarlyStopper;
import edu.neu.ccs.pyramid.util.ListUtil;
import edu.neu.ccs.pyramid.util.Pair;
import edu.neu.ccs.pyramid.util.PrintUtil;
import edu.neu.ccs.pyramid.util.Serialization;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.time.StopWatch;

import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.nio.file.Paths;
import java.util.*;
import java.util.logging.FileHandler;
import java.util.logging.Logger;
import java.util.logging.SimpleFormatter;
import java.util.stream.Collectors;
import java.util.stream.IntStream;

public class BRLREN {
    private static boolean VERBOSE = false;

    public static void main(Config config) throws Exception{
        Logger logger = Logger.getAnonymousLogger();
        String logFile = config.getString("output.log");
        FileHandler fileHandler = null;
        if (!logFile.isEmpty()){
            new File(logFile).getParentFile().mkdirs();
            //todo should append?
            fileHandler = new FileHandler(logFile, true);
            java.util.logging.Formatter formatter = new SimpleFormatter();
            fileHandler.setFormatter(formatter);
            logger.addHandler(fileHandler);
            logger.setUseParentHandlers(false);
        }


//        logger.info(config.toString());

        new File(config.getString("output.dir")).mkdirs();

        if (config.getBoolean("tune")){
            logger.info("============================================================");
            logger.info("Start hyper parameter tuning");
            StopWatch stopWatch = new StopWatch();
            stopWatch.start();
            List<TuneResult> tuneResults = new ArrayList<>();
            List<MultiLabelClfDataSet> dataSets = loadTrainValidData(config, logger);
            List<Double> penalties = config.getDoubles("tune.penalty.candidates");
            List<Double> l1Ratioes = config.getDoubles("tune.l1Ratio.candidates");
            List<Integer> components = config.getIntegers("tune.numComponents.candidates");
            for (double penalty: penalties){
                for (double l1Ratio : l1Ratioes) {
                    for (int component: components){
                        StopWatch stopWatch1 = new StopWatch();
                        stopWatch1.start();
                        HyperParameters hyperParameters = new HyperParameters();
                        hyperParameters.numComponents = component;
                        hyperParameters.l1Ratio = l1Ratio;
                        hyperParameters.penalty = penalty;
                        logger.info("---------------------------");
                        logger.info("Trying hyper parameters:");
                        logger.info("train.numComponents = "+hyperParameters.numComponents);
                        logger.info("train.penalty = "+hyperParameters.penalty);
                        logger.info("train.l1Ratio = "+hyperParameters.l1Ratio);
                        TuneResult tuneResult = tune(config, hyperParameters, dataSets.get(0), dataSets.get(1), logger);
                        logger.info("Found optimal train.iterations = "+tuneResult.hyperParameters.iterations);
                        logger.info("Validation performance = "+tuneResult.performance);
                        tuneResults.add(tuneResult);
                        logger.info("Time spent on trying this set of hyper parameters = "+stopWatch1);
                    }
                }
            }

            Comparator<TuneResult> comparator = Comparator.comparing(res->res.performance);

            TuneResult best;
            String predictTarget = config.getString("tune.targetMetric");
            switch (predictTarget){
                case "instance_set_accuracy":
                    best = tuneResults.stream().max(comparator).get();
                    break;
                case "instance_f1":
                    best = tuneResults.stream().max(comparator).get();
                    break;
                case "instance_hamming_loss":
                    best = tuneResults.stream().min(comparator).get();
                    break;
                case "label_map":
                    best = tuneResults.stream().max(comparator).get();
                    break;
                default:
                    throw new IllegalArgumentException("tune.targetMetric should be instance_set_accuracy, instance_f1 or instance_hamming_loss");
            }


            logger.info("---------------------------");
            logger.info("Hyper parameter tuning done.");
            logger.info("Time spent on entire hyper parameter tuning = "+stopWatch);
            logger.info("Best validation performance = "+best.performance);
            logger.info("Best hyper parameters:");
            logger.info("train.numComponents = "+best.hyperParameters.numComponents);
            logger.info("train.penalty = "+best.hyperParameters.penalty);
            logger.info("train.l1Ratio = "+best.hyperParameters.l1Ratio);
            logger.info("train.iterations = "+best.hyperParameters.iterations);
            Config tunedHypers = best.hyperParameters.asConfig();
            tunedHypers.store(Paths.get(config.getString("output.dir"),"model_predictions",config.getString("output.modelFolder"),"models","tuned_hyper_parameters.properties").toFile());
            logger.info("Tuned hyper parameters saved to "+Paths.get(config.getString("output.dir"),"model_predictions",config.getString("output.modelFolder"),"models","tuned_hyper_parameters.properties").toFile().getAbsolutePath());
            logger.info("============================================================");
        }

        if (config.getBoolean("train")){
            logger.info("============================================================");
            if (config.getBoolean("train.useTunedHyperParameters")){
                File hyperFile = Paths.get(config.getString("output.dir"),"model_predictions",config.getString("output.modelFolder"),"models","tuned_hyper_parameters.properties").toFile();
                if (!hyperFile.exists()){
                    logger.info("train.useTunedHyperParameters is set to true. But no tuned hyper parameters can be found in the output directory.");
                    logger.info("Please either run hyper parameter tuning, or provide hyper parameters manually and set train.useTunedHyperParameters=false.");
                    System.exit(1);
                }
                Config tunedHypers = new Config(hyperFile);
                HyperParameters hyperParameters = new HyperParameters(tunedHypers);
                logger.info("Start training with tuned hyper parameters:");
                logger.info("train.numComponents = "+hyperParameters.numComponents);
                logger.info("train.penalty = "+hyperParameters.penalty);
                logger.info("train.l1Ratio = "+hyperParameters.l1Ratio);
                logger.info("train.iterations = "+hyperParameters.iterations);


                MultiLabelClfDataSet trainSet = loadTrainData(config);
                train(config, hyperParameters, trainSet, logger);
            } else {
                HyperParameters hyperParameters = new HyperParameters(config);
                logger.info("Start training with given hyper parameters:");
                logger.info("train.numComponents = "+hyperParameters.numComponents);
                logger.info("train.penalty = "+hyperParameters.penalty);
                logger.info("train.l1Ratio = "+hyperParameters.l1Ratio);
                logger.info("train.iterations = "+hyperParameters.iterations);

                MultiLabelClfDataSet trainSet = loadTrainData(config);
                train(config, hyperParameters, trainSet, logger);
            }
            logger.info("============================================================");
        }

        }


        if (fileHandler!=null){
            fileHandler.close();
        }
    }

        if (args.length != 1) {
            throw new IllegalArgumentException("Please specify a properties file.");
        }
        
        
        Config config = new Config(args[0]);

        main(config);


    }


    private static TuneResult tune(Config config, HyperParameters hyperParameters, MultiLabelClfDataSet trainSet, MultiLabelClfDataSet validSet,
                                   Logger logger) throws Exception{

        CBM cbm = newCBM(config, trainSet, hyperParameters, logger);
        EarlyStopper earlyStopper = loadNewEarlyStopper(config);

        ENCBMOptimizer optimizer = getOptimizer(config, hyperParameters, cbm, trainSet);
        if (config.getBoolean("train.randomInitialize")) {
            optimizer.randInitialize();
        } else {
            optimizer.initialize();
        }

        MultiLabelClassifier classifier;
        String predictTarget = config.getString("tune.targetMetric");
        switch (predictTarget){
            case "instance_set_accuracy":
                AccPredictor accPredictor = new AccPredictor(cbm);
                accPredictor.setComponentContributionThreshold(config.getDouble("predict.piThreshold"));
                classifier = accPredictor;
                break;
            case "instance_f1":
                PluginF1 pluginF1 = new PluginF1(cbm);
                List<MultiLabel> support = DataSetUtil.gatherMultiLabels(trainSet);
                pluginF1.setSupport(support);
                pluginF1.setPiThreshold(config.getDouble("predict.piThreshold"));
                classifier = pluginF1;
                break;
            case "instance_hamming_loss":
                MarginalPredictor marginalPredictor = new MarginalPredictor(cbm);
                marginalPredictor.setPiThreshold(config.getDouble("predict.piThreshold"));
                classifier = marginalPredictor;
                break;

            case "label_map":
                AccPredictor accPredictor2 = new AccPredictor(cbm);
                accPredictor2.setComponentContributionThreshold(config.getDouble("predict.piThreshold"));
                classifier = accPredictor2;
                break;
            default:
                throw new IllegalArgumentException("predictTarget should be instance_set_accuracy, instance_f1 or instance_hamming_loss");
        }

        int interval = config.getInt("tune.monitorInterval");

        for (int iter = 1; true; iter++){

            if (VERBOSE){
                logger.info("iteration "+iter );
            }


            optimizer.iterate();

            if (iter%interval==0){


                MLMeasures validMeasures = new MLMeasures(classifier,validSet);
                if (VERBOSE){
                    logger.info("validation performance with "+predictTarget+" optimal predictor:");
                    logger.info(validMeasures.toString());
                }

                switch (predictTarget){
                    case "instance_set_accuracy":
                        earlyStopper.add(iter,validMeasures.getInstanceAverage().getAccuracy());
                        break;
                    case "instance_f1":
                        earlyStopper.add(iter,validMeasures.getInstanceAverage().getF1());
                        break;
                    case "instance_hamming_loss":
                        earlyStopper.add(iter,validMeasures.getInstanceAverage().getHammingLoss());
                        break;
                    case "label_map":
                        List<MultiLabel> support = DataSetUtil.gatherMultiLabels(trainSet);
                        double map = MAP.mapBySupport(cbm, validSet,support);
                        earlyStopper.add(iter,map);
                        break;
                    default:
                        throw new IllegalArgumentException("predictTarget should be instance_set_accuracy or instance_f1");
                }

                if (earlyStopper.shouldStop()){
                    if (VERBOSE){
                        logger.info("Early Stopper: the training should stop now!");
                if (earlyStopper.shouldStop()){
                    if (VERBOSE){
                        logger.info("Early Stopper: the training should stop now!");
                        logger.info("Early Stopper: best iteration found = "+earlyStopper.getBestIteration());
            }
        }

        if (VERBOSE){
            logger.info("done!");
        }

        hyperParameters.iterations = earlyStopper.getBestIteration();
        TuneResult tuneResult = new TuneResult();
        tuneResult.hyperParameters = hyperParameters;
        tuneResult.performance = earlyStopper.getBestValue();
        return tuneResult;

    }


    private static void train(Config config, HyperParameters hyperParameters, MultiLabelClfDataSet trainSet,
                              Logger logger) throws Exception{

        List<Integer> unobservedLabels = DataSetUtil.unobservedLabels(trainSet);

        if (!unobservedLabels.isEmpty()){
            logger.info("The following labels do not actually appear in the training set and therefore cannot be learned:");
            logger.info(ListUtil.toSimpleString(unobservedLabels));
            FileUtils.writeStringToFile(Paths.get(config.getString("output.dir"),"model_predictions",config.getString("output.modelFolder"),"analysis","unobserved_labels.txt").toFile(), ListUtil.toSimpleString(unobservedLabels));
        }
        String output = config.getString("output.dir");


        StopWatch stopWatch = new StopWatch();
        stopWatch.start();
        CBM cbm = newCBM(config,trainSet, hyperParameters, logger);

        ENCBMOptimizer optimizer = getOptimizer(config, hyperParameters, cbm, trainSet);
        logger.info("Initializing the model");
        if (config.getBoolean("train.randomInitialize")) {
            optimizer.randInitialize();
        } else {
            optimizer.initialize();
        }
        logger.info("Initialization done");

        for (int iter=1;iter<=hyperParameters.iterations;iter++){
            logger.info("Training progress: iteration "+iter );
            optimizer.iterate();
        }

        logger.info("training done!");
        logger.info("time spent on training = "+stopWatch);

        Serialization.serialize(cbm, Paths.get(output,"model_predictions",config.getString("output.modelFolder"),"models","classifier"));
        List<MultiLabel> support = DataSetUtil.gatherMultiLabels(trainSet);
        Serialization.serialize(support, Paths.get(output,"model_predictions",config.getString("output.modelFolder"),"models","support"));

        featureImportance(config, cbm, trainSet.getFeatureList(), trainSet.getLabelTranslator(),logger);
    }



        CBM bestModel = (CBM) Serialization.deserialize(Paths.get(output,"model_predictions",config.getString("output.modelFolder"),"models","classifier"));

        boolean topFeaturesToFile = true;

        if (topFeaturesToFile){
            File analysisFolder = Paths.get(output, "model_predictions",config.getString("output.modelFolder"),"analysis").toFile();
            analysisFolder.mkdirs();
            logger.info("start writing top features");
            List<TopFeatures> topFeaturesList = IntStream.range(0,bestModel.getNumClasses())
                    .mapToObj(k ->topFeatures(bestModel, trainSet.getFeatureList(), trainSet.getLabelTranslator(), k, 100)).collect(Collectors.toList());
            ObjectMapper mapper = new ObjectMapper();
            String file = "top_features.json";
            mapper.writeValue(new File(analysisFolder,file), topFeaturesList);

            StringBuilder sb = new StringBuilder();
            for (int l=0;l<bestModel.getNumClasses();l++){
                sb.append("-------------------------").append("\n");
                sb.append(bestModel.getLabelTranslator().toExtLabel(l)).append(":").append("\n");
                for (Feature feature: topFeaturesList.get(l).getTopFeatures()){
                    sb.append(feature.simpleString()).append(", ");
                }
                sb.append("\n");
            }
            FileUtils.writeStringToFile(new File(analysisFolder, "top_features.txt"), sb.toString());

            logger.info("finish writing top features");
        }

    }


    private static TopFeatures topFeatures(CBM cbm, FeatureList featureList, LabelTranslator mlLabelTranslator, int classIndex,
                                           int limit){
        LogisticRegression logisticRegression = (LogisticRegression) cbm.getBinaryClassifiers()[0][classIndex];
        logisticRegression.setFeatureList(featureList);
        List<String> labels = new ArrayList<>();
        labels.add("not_"+mlLabelTranslator.toExtLabel(classIndex));
        labels.add(mlLabelTranslator.toExtLabel(classIndex));
        LabelTranslator labelTranslator = new LabelTranslator(labels);
        logisticRegression.setLabelTranslator(labelTranslator);
        TopFeatures topFeatures = LogisticRegressionInspector.topFeatures(logisticRegression, 1,limit);
        return topFeatures;
    }


    //todo currently only for br
    private static void featureImportance(Config config, CBM cbm, FeatureList featureList, LabelTranslator mlLabelTranslator,
                                          Logger logger) throws Exception{

        logger.info("number of selected features in all labels (union)= "+CBMInspector.usedFeatures(cbm).size());
        int[] featuresByEach = CBMInspector.usedFeaturesByEachLabel(cbm);
        double average = Arrays.stream(featuresByEach).average().getAsDouble();
        logger.info("average number of selected features in each label ="+average);

        StringBuilder sbcount = new StringBuilder();
        for (int l=0;l<featuresByEach.length;l++){
            sbcount.append(mlLabelTranslator.toExtLabel(l)).append(":").append(featuresByEach[l]).append("\n");
        }

        String output = config.getString("output.dir");
        Paths.get(output, "model_predictions",config.getString("output.modelFolder"),"analysis").toFile().mkdirs();
        try (BufferedWriter bw = new BufferedWriter(new FileWriter(Paths.get(output, "model_predictions",config.getString("output.modelFolder"),"analysis","top_features.txt").toFile()))){
            for (int l=0;l<cbm.getNumClasses();l++){
                if (cbm.getBinaryClassifiers()[0][l] instanceof LogisticRegression){
                    LogisticRegression logisticRegression = (LogisticRegression) cbm.getBinaryClassifiers()[0][l];
                    logisticRegression.setFeatureList(featureList);
                    List<String> labels = new ArrayList<>();
                    labels.add("not_"+mlLabelTranslator.toExtLabel(l));
                    labels.add(mlLabelTranslator.toExtLabel(l));
                    LabelTranslator labelTranslator = new LabelTranslator(labels);
                    logisticRegression.setLabelTranslator(labelTranslator);
                    TopFeatures topFeatures = LogisticRegressionInspector.topFeatures(logisticRegression, 1,Integer.MAX_VALUE);
                    bw.write("label "+l+" ("+mlLabelTranslator.toExtLabel(l)+")");
                    bw.write(": ");
                    for (int f=0;f<topFeatures.getTopFeatures().size();f++){
                        Feature feature = topFeatures.getTopFeatures().get(f);
                        double utility = topFeatures.getUtilities().get(f);
                        if (utility!=0){
                            bw.write(""+feature.getIndex());
//                        bw.write(" (");
//                        bw.write(feature.getName());
//                        bw.write(")");
                            bw.write(":");
                            bw.write(""+utility);
                            bw.write(", ");
                        }

                    }
                    bw.write("\n");
                }
            }
        }
        logger.info("feature count in each label is saved to the file "+Paths.get(output, "model_predictions",config.getString("output.modelFolder"),"analysis","feature_count_in_each_label.txt").toFile().getAbsolutePath());
        FileUtils.writeStringToFile(Paths.get(output, "model_predictions",config.getString("output.modelFolder"),"analysis","feature_count_in_each_label.txt").toFile(),sbcount.toString());

    }

    private static ENCBMOptimizer getOptimizer(Config config, HyperParameters hyperParameters, CBM cbm, MultiLabelClfDataSet trainSet){
        ENCBMOptimizer optimizer = new ENCBMOptimizer(cbm, trainSet);

        if (config.getBoolean("train.useInstanceWeights")){
            double[] instanceWeights = loadInstanceWeights(config);
            optimizer.setInstanceWeights(instanceWeights);
        }

        optimizer.setLineSearch(config.getBoolean("train.elasticnet.lineSearch"));
        optimizer.setRegularizationBinary(hyperParameters.penalty);
        optimizer.setRegularizationMultiClass(hyperParameters.penalty);
        optimizer.setL1RatioBinary(hyperParameters.l1Ratio);
        optimizer.setL1RatioMultiClass(hyperParameters.l1Ratio);
        optimizer.setActiveSet(config.getBoolean("train.elasticnet.activeSet"));

        optimizer.setBinaryUpdatesPerIter(config.getInt("train.updatesPerIteration"));
        optimizer.setMulticlassUpdatesPerIter(config.getInt("train.updatesPerIteration"));
        optimizer.setSkipDataThreshold(config.getDouble("train.skipDataThreshold"));
        optimizer.setSkipLabelThreshold(config.getDouble("train.skipLabelThreshold"));
        optimizer.setMaxNumLinearRegUpdates(config.getInt("train.maxNumLinearRegUpdates"));
//

        return optimizer;
    }


    private static CBM newCBM(Config config, MultiLabelClfDataSet trainSet, HyperParameters hyperParameters,
                              Logger logger){

        CBM cbm;
    }



    private static EarlyStopper loadNewEarlyStopper(Config config){
        String earlyStopMetric = config.getString("tune.targetMetric");
        int patience = config.getInt("tune.earlyStop.patience");
        EarlyStopper.Goal earlyStopGoal = null;
        switch (earlyStopMetric){
            case "instance_set_accuracy":
                earlyStopGoal = EarlyStopper.Goal.MAXIMIZE;
                break;
            case "instance_f1":
                earlyStopGoal = EarlyStopper.Goal.MAXIMIZE;
                break;
            case "instance_hamming_loss":
                earlyStopGoal = EarlyStopper.Goal.MINIMIZE;
                break;
            case "label_map":
                earlyStopGoal = EarlyStopper.Goal.MAXIMIZE;
                break;
            default:
                throw new IllegalArgumentException("unsupported tune.targetMetric "+earlyStopMetric);
        }

        EarlyStopper earlyStopper = new EarlyStopper(earlyStopGoal,patience);
        earlyStopper.setMinimumIterations(config.getInt("tune.earlyStop.minIterations"));
        return earlyStopper;
    }

    private static List<MultiLabelClfDataSet> loadTrainValidData(Config config, Logger logger) throws Exception{
        String validPath = config.getString("input.validData");
        List<MultiLabelClfDataSet> datasets = new ArrayList<>();
        MultiLabelClfDataSet trainSet = TRECFormat.loadMultiLabelClfDataSetAutoSparseSequential(config.getString("input.trainData"));

        if (validPath.isEmpty()){
            logger.info("No external validation data is provided. Use random 20% of the training data for validation.");
            Pair<MultiLabelClfDataSet, MultiLabelClfDataSet> dataSetPair = DataSetUtil.splitToTrainValidation(trainSet,0.8);
            MultiLabelClfDataSet subTrain = dataSetPair.getFirst();
            MultiLabelClfDataSet validSet  = dataSetPair.getSecond();
            datasets.add(subTrain);
            datasets.add(validSet);
        } else {
            MultiLabelClfDataSet validSet = TRECFormat.loadMultiLabelClfDataSetAutoSparseSequential(config.getString("input.validData"));
            datasets.add(trainSet);
            datasets.add(validSet);
        }
        return datasets;
    }

    private static MultiLabelClfDataSet loadTrainData(Config config) throws Exception{
            case "false":
                cbm.setAllowEmpty(false);
                break;
            case "auto":
                Set<MultiLabel> seen = DataSetUtil.gatherMultiLabels(trainSet).stream().collect(Collectors.toSet());
                MultiLabel empty = new MultiLabel();
                if (seen.contains(empty)){
                    cbm.setAllowEmpty(true);
                    if (VERBOSE){
                        logger.info("training set contains empty labels, automatically set predict.allowEmpty = true");
                    }

                } else {
                    cbm.setAllowEmpty(false);
                    if (VERBOSE){
    private static class HyperParameters{
        double penalty;
        double l1Ratio;
        int iterations;
        int numComponents;

        HyperParameters() {
        }

        HyperParameters(Config config) {
            penalty = config.getDouble("train.penalty");
            l1Ratio = config.getDouble("train.l1Ratio");
            iterations = config.getInt("train.iterations");
            numComponents = config.getInt("train.numComponents");
        }

    private static EarlyStopper loadNewEarlyStopper(){
            Config config = new Config();
            config.setDouble("train.penalty", penalty);
            config.setDouble("train.l1Ratio", l1Ratio);
            config.setInt("train.iterations", iterations);
            config.setInt("train.numComponents", numComponents);
            return config;
        }


    }

    private static class TuneResult{
        HyperParameters hyperParameters;
        double performance;
    }

    private static boolean containsNovelClass(MultiLabel multiLabel, List<Integer> novelLabels){
        for (int l:novelLabels){
        MultiLabelClfDataSet trainSet = TRECFormat.loadMultiLabelClfDataSetAutoSparseSequential(config.getString("input.trainData"));
        return trainSet;
    }

    private static double[] loadInstanceWeights(Config config){
        File file = new File(config.getString("input.trainData"),"instance_weights.txt");
        double[] weights = new double[0];
        try {
            weights = FileUtils.readLines(file).stream().mapToDouble(Double::parseDouble).toArray();
        } catch (IOException e) {
            e.printStackTrace();
        }
        return weights;
    }

    private static class HyperParameters{
        double penalty;
        double l1Ratio;
        int numComponents;

        HyperParameters() {
        }

        HyperParameters(Config config) {
            penalty = config.getDouble("train.penalty");
            l1Ratio = config.getDouble("train.l1Ratio");
            numComponents = config.getInt("train.numComponents");
        }

        Config asConfig(){
            Config config = new Config();
            config.setDouble("train.penalty", penalty);
            config.setDouble("train.l1Ratio", l1Ratio);
            config.setInt("train.numComponents", numComponents);
            return config;
        }


    }


    private static boolean containsNovelClass(MultiLabel multiLabel, List<Integer> novelLabels){
        for (int l:novelLabels){
            if (multiLabel.matchClass(l)){
                return true;
            }
        }
        return false;
    }
}

<MSG> early stopping

<DFF> @@ -56,107 +56,16 @@ public class BRLREN {
 
         new File(config.getString("output.dir")).mkdirs();
 
-        if (config.getBoolean("tune")){
-            logger.info("============================================================");
-            logger.info("Start hyper parameter tuning");
-            StopWatch stopWatch = new StopWatch();
-            stopWatch.start();
-            List<TuneResult> tuneResults = new ArrayList<>();
-            List<MultiLabelClfDataSet> dataSets = loadTrainValidData(config, logger);
-            List<Double> penalties = config.getDoubles("tune.penalty.candidates");
-            List<Double> l1Ratioes = config.getDoubles("tune.l1Ratio.candidates");
-            List<Integer> components = config.getIntegers("tune.numComponents.candidates");
-            for (double penalty: penalties){
-                for (double l1Ratio : l1Ratioes) {
-                    for (int component: components){
-                        StopWatch stopWatch1 = new StopWatch();
-                        stopWatch1.start();
-                        HyperParameters hyperParameters = new HyperParameters();
-                        hyperParameters.numComponents = component;
-                        hyperParameters.l1Ratio = l1Ratio;
-                        hyperParameters.penalty = penalty;
-                        logger.info("---------------------------");
-                        logger.info("Trying hyper parameters:");
-                        logger.info("train.numComponents = "+hyperParameters.numComponents);
-                        logger.info("train.penalty = "+hyperParameters.penalty);
-                        logger.info("train.l1Ratio = "+hyperParameters.l1Ratio);
-                        TuneResult tuneResult = tune(config, hyperParameters, dataSets.get(0), dataSets.get(1), logger);
-                        logger.info("Found optimal train.iterations = "+tuneResult.hyperParameters.iterations);
-                        logger.info("Validation performance = "+tuneResult.performance);
-                        tuneResults.add(tuneResult);
-                        logger.info("Time spent on trying this set of hyper parameters = "+stopWatch1);
-                    }
-                }
-            }
 
-            Comparator<TuneResult> comparator = Comparator.comparing(res->res.performance);
-
-            TuneResult best;
-            String predictTarget = config.getString("tune.targetMetric");
-            switch (predictTarget){
-                case "instance_set_accuracy":
-                    best = tuneResults.stream().max(comparator).get();
-                    break;
-                case "instance_f1":
-                    best = tuneResults.stream().max(comparator).get();
-                    break;
-                case "instance_hamming_loss":
-                    best = tuneResults.stream().min(comparator).get();
-                    break;
-                case "label_map":
-                    best = tuneResults.stream().max(comparator).get();
-                    break;
-                default:
-                    throw new IllegalArgumentException("tune.targetMetric should be instance_set_accuracy, instance_f1 or instance_hamming_loss");
-            }
-
-
-            logger.info("---------------------------");
-            logger.info("Hyper parameter tuning done.");
-            logger.info("Time spent on entire hyper parameter tuning = "+stopWatch);
-            logger.info("Best validation performance = "+best.performance);
-            logger.info("Best hyper parameters:");
-            logger.info("train.numComponents = "+best.hyperParameters.numComponents);
-            logger.info("train.penalty = "+best.hyperParameters.penalty);
-            logger.info("train.l1Ratio = "+best.hyperParameters.l1Ratio);
-            logger.info("train.iterations = "+best.hyperParameters.iterations);
-            Config tunedHypers = best.hyperParameters.asConfig();
-            tunedHypers.store(Paths.get(config.getString("output.dir"),"model_predictions",config.getString("output.modelFolder"),"models","tuned_hyper_parameters.properties").toFile());
-            logger.info("Tuned hyper parameters saved to "+Paths.get(config.getString("output.dir"),"model_predictions",config.getString("output.modelFolder"),"models","tuned_hyper_parameters.properties").toFile().getAbsolutePath());
-            logger.info("============================================================");
-        }
 
         if (config.getBoolean("train")){
             logger.info("============================================================");
-            if (config.getBoolean("train.useTunedHyperParameters")){
-                File hyperFile = Paths.get(config.getString("output.dir"),"model_predictions",config.getString("output.modelFolder"),"models","tuned_hyper_parameters.properties").toFile();
-                if (!hyperFile.exists()){
-                    logger.info("train.useTunedHyperParameters is set to true. But no tuned hyper parameters can be found in the output directory.");
-                    logger.info("Please either run hyper parameter tuning, or provide hyper parameters manually and set train.useTunedHyperParameters=false.");
-                    System.exit(1);
-                }
-                Config tunedHypers = new Config(hyperFile);
-                HyperParameters hyperParameters = new HyperParameters(tunedHypers);
-                logger.info("Start training with tuned hyper parameters:");
-                logger.info("train.numComponents = "+hyperParameters.numComponents);
-                logger.info("train.penalty = "+hyperParameters.penalty);
-                logger.info("train.l1Ratio = "+hyperParameters.l1Ratio);
-                logger.info("train.iterations = "+hyperParameters.iterations);
-
-
-                MultiLabelClfDataSet trainSet = loadTrainData(config);
-                train(config, hyperParameters, trainSet, logger);
-            } else {
-                HyperParameters hyperParameters = new HyperParameters(config);
-                logger.info("Start training with given hyper parameters:");
-                logger.info("train.numComponents = "+hyperParameters.numComponents);
-                logger.info("train.penalty = "+hyperParameters.penalty);
-                logger.info("train.l1Ratio = "+hyperParameters.l1Ratio);
-                logger.info("train.iterations = "+hyperParameters.iterations);
-
-                MultiLabelClfDataSet trainSet = loadTrainData(config);
-                train(config, hyperParameters, trainSet, logger);
-            }
+            HyperParameters hyperParameters = new HyperParameters(config);
+
+
+            MultiLabelClfDataSet trainSet = loadTrainData(config);
+            MultiLabelClfDataSet validSet = loadValidData(config);
+            train(config, hyperParameters, trainSet, validSet, logger);
             logger.info("============================================================");
         }
 
@@ -171,98 +80,63 @@ public class BRLREN {
         if (args.length != 1) {
             throw new IllegalArgumentException("Please specify a properties file.");
         }
-        
-        
+
         Config config = new Config(args[0]);
 
         main(config);
+    }
 
 
-    }
 
+    private static void train(Config config, HyperParameters hyperParameters, MultiLabelClfDataSet trainSet, MultiLabelClfDataSet validSet,
+                              Logger logger) throws Exception{
 
-    private static TuneResult tune(Config config, HyperParameters hyperParameters, MultiLabelClfDataSet trainSet, MultiLabelClfDataSet validSet,
-                                   Logger logger) throws Exception{
+        List<Integer> unobservedLabels = DataSetUtil.unobservedLabels(trainSet);
 
-        CBM cbm = newCBM(config, trainSet, hyperParameters, logger);
-        EarlyStopper earlyStopper = loadNewEarlyStopper(config);
+        if (!unobservedLabels.isEmpty()){
+            logger.info("The following labels do not actually appear in the training set and therefore cannot be learned:");
+            logger.info(ListUtil.toSimpleString(unobservedLabels));
+            FileUtils.writeStringToFile(Paths.get(config.getString("output.dir"),"model_predictions",config.getString("output.modelFolder"),"analysis","unobserved_labels.txt").toFile(), ListUtil.toSimpleString(unobservedLabels));
+        }
+        String output = config.getString("output.dir");
+        EarlyStopper earlyStopper = loadNewEarlyStopper();
+
+        StopWatch stopWatch = new StopWatch();
+        stopWatch.start();
+        CBM cbm = newCBM(config,trainSet, hyperParameters, logger);
 
         ENCBMOptimizer optimizer = getOptimizer(config, hyperParameters, cbm, trainSet);
+        logger.info("Initializing the model");
         if (config.getBoolean("train.randomInitialize")) {
             optimizer.randInitialize();
         } else {
             optimizer.initialize();
         }
+        logger.info("Initialization done");
 
-        MultiLabelClassifier classifier;
-        String predictTarget = config.getString("tune.targetMetric");
-        switch (predictTarget){
-            case "instance_set_accuracy":
-                AccPredictor accPredictor = new AccPredictor(cbm);
-                accPredictor.setComponentContributionThreshold(config.getDouble("predict.piThreshold"));
-                classifier = accPredictor;
-                break;
-            case "instance_f1":
-                PluginF1 pluginF1 = new PluginF1(cbm);
-                List<MultiLabel> support = DataSetUtil.gatherMultiLabels(trainSet);
-                pluginF1.setSupport(support);
-                pluginF1.setPiThreshold(config.getDouble("predict.piThreshold"));
-                classifier = pluginF1;
-                break;
-            case "instance_hamming_loss":
-                MarginalPredictor marginalPredictor = new MarginalPredictor(cbm);
-                marginalPredictor.setPiThreshold(config.getDouble("predict.piThreshold"));
-                classifier = marginalPredictor;
-                break;
-
-            case "label_map":
-                AccPredictor accPredictor2 = new AccPredictor(cbm);
-                accPredictor2.setComponentContributionThreshold(config.getDouble("predict.piThreshold"));
-                classifier = accPredictor2;
-                break;
-            default:
-                throw new IllegalArgumentException("predictTarget should be instance_set_accuracy, instance_f1 or instance_hamming_loss");
-        }
-
-        int interval = config.getInt("tune.monitorInterval");
-
-        for (int iter = 1; true; iter++){
 
-            if (VERBOSE){
-                logger.info("iteration "+iter );
-            }
+        AccPredictor accPredictor = new AccPredictor(cbm);
+        accPredictor.setComponentContributionThreshold(config.getDouble("predict.piThreshold"));
+        CBM bestModel = null;
 
+        int interval = 1;
+        for (int iter=1;true;iter++){
 
+            logger.info("Training progress: iteration "+iter );
             optimizer.iterate();
 
             if (iter%interval==0){
-
-
-                MLMeasures validMeasures = new MLMeasures(classifier,validSet);
+                MLMeasures validMeasures = new MLMeasures(accPredictor,validSet);
                 if (VERBOSE){
-                    logger.info("validation performance with "+predictTarget+" optimal predictor:");
+                    logger.info("validation performance");
                     logger.info(validMeasures.toString());
                 }
-
-                switch (predictTarget){
-                    case "instance_set_accuracy":
-                        earlyStopper.add(iter,validMeasures.getInstanceAverage().getAccuracy());
-                        break;
-                    case "instance_f1":
-                        earlyStopper.add(iter,validMeasures.getInstanceAverage().getF1());
-                        break;
-                    case "instance_hamming_loss":
-                        earlyStopper.add(iter,validMeasures.getInstanceAverage().getHammingLoss());
-                        break;
-                    case "label_map":
-                        List<MultiLabel> support = DataSetUtil.gatherMultiLabels(trainSet);
-                        double map = MAP.mapBySupport(cbm, validSet,support);
-                        earlyStopper.add(iter,map);
-                        break;
-                    default:
-                        throw new IllegalArgumentException("predictTarget should be instance_set_accuracy or instance_f1");
+                earlyStopper.add(iter,validMeasures.getInstanceAverage().getAccuracy());
+                if (earlyStopper.getBestIteration()==iter){
+                    bestModel = (CBM)Serialization.deepCopy(cbm);
                 }
 
+
                 if (earlyStopper.shouldStop()){
                     if (VERBOSE){
                         logger.info("Early Stopper: the training should stop now!");
@@ -273,58 +147,14 @@ public class BRLREN {
             }
         }
 
-        if (VERBOSE){
-            logger.info("done!");
-        }
-
-        hyperParameters.iterations = earlyStopper.getBestIteration();
-        TuneResult tuneResult = new TuneResult();
-        tuneResult.hyperParameters = hyperParameters;
-        tuneResult.performance = earlyStopper.getBestValue();
-        return tuneResult;
-
-    }
-
-
-    private static void train(Config config, HyperParameters hyperParameters, MultiLabelClfDataSet trainSet,
-                              Logger logger) throws Exception{
-
-        List<Integer> unobservedLabels = DataSetUtil.unobservedLabels(trainSet);
-
-        if (!unobservedLabels.isEmpty()){
-            logger.info("The following labels do not actually appear in the training set and therefore cannot be learned:");
-            logger.info(ListUtil.toSimpleString(unobservedLabels));
-            FileUtils.writeStringToFile(Paths.get(config.getString("output.dir"),"model_predictions",config.getString("output.modelFolder"),"analysis","unobserved_labels.txt").toFile(), ListUtil.toSimpleString(unobservedLabels));
-        }
-        String output = config.getString("output.dir");
-
-
-        StopWatch stopWatch = new StopWatch();
-        stopWatch.start();
-        CBM cbm = newCBM(config,trainSet, hyperParameters, logger);
-
-        ENCBMOptimizer optimizer = getOptimizer(config, hyperParameters, cbm, trainSet);
-        logger.info("Initializing the model");
-        if (config.getBoolean("train.randomInitialize")) {
-            optimizer.randInitialize();
-        } else {
-            optimizer.initialize();
-        }
-        logger.info("Initialization done");
-
-        for (int iter=1;iter<=hyperParameters.iterations;iter++){
-            logger.info("Training progress: iteration "+iter );
-            optimizer.iterate();
-        }
-
         logger.info("training done!");
         logger.info("time spent on training = "+stopWatch);
 
-        Serialization.serialize(cbm, Paths.get(output,"model_predictions",config.getString("output.modelFolder"),"models","classifier"));
+        Serialization.serialize(bestModel, Paths.get(output,"model_predictions",config.getString("output.modelFolder"),"models","classifier"));
         List<MultiLabel> support = DataSetUtil.gatherMultiLabels(trainSet);
         Serialization.serialize(support, Paths.get(output,"model_predictions",config.getString("output.modelFolder"),"models","support"));
 
-        featureImportance(config, cbm, trainSet.getFeatureList(), trainSet.getLabelTranslator(),logger);
+        featureImportance(config, bestModel, trainSet.getFeatureList(), trainSet.getLabelTranslator(),logger);
     }
 
 
@@ -459,51 +289,17 @@ public class BRLREN {
     }
 
 
-
-    private static EarlyStopper loadNewEarlyStopper(Config config){
-        String earlyStopMetric = config.getString("tune.targetMetric");
-        int patience = config.getInt("tune.earlyStop.patience");
-        EarlyStopper.Goal earlyStopGoal = null;
-        switch (earlyStopMetric){
-            case "instance_set_accuracy":
-                earlyStopGoal = EarlyStopper.Goal.MAXIMIZE;
-                break;
-            case "instance_f1":
-                earlyStopGoal = EarlyStopper.Goal.MAXIMIZE;
-                break;
-            case "instance_hamming_loss":
-                earlyStopGoal = EarlyStopper.Goal.MINIMIZE;
-                break;
-            case "label_map":
-                earlyStopGoal = EarlyStopper.Goal.MAXIMIZE;
-                break;
-            default:
-                throw new IllegalArgumentException("unsupported tune.targetMetric "+earlyStopMetric);
-        }
-
-        EarlyStopper earlyStopper = new EarlyStopper(earlyStopGoal,patience);
-        earlyStopper.setMinimumIterations(config.getInt("tune.earlyStop.minIterations"));
+    private static EarlyStopper loadNewEarlyStopper(){
+        int patience = 5;
+        EarlyStopper earlyStopper = new EarlyStopper(EarlyStopper.Goal.MAXIMIZE,patience);
+        earlyStopper.setMinimumIterations(5);
         return earlyStopper;
     }
 
-    private static List<MultiLabelClfDataSet> loadTrainValidData(Config config, Logger logger) throws Exception{
-        String validPath = config.getString("input.validData");
-        List<MultiLabelClfDataSet> datasets = new ArrayList<>();
-        MultiLabelClfDataSet trainSet = TRECFormat.loadMultiLabelClfDataSetAutoSparseSequential(config.getString("input.trainData"));
 
-        if (validPath.isEmpty()){
-            logger.info("No external validation data is provided. Use random 20% of the training data for validation.");
-            Pair<MultiLabelClfDataSet, MultiLabelClfDataSet> dataSetPair = DataSetUtil.splitToTrainValidation(trainSet,0.8);
-            MultiLabelClfDataSet subTrain = dataSetPair.getFirst();
-            MultiLabelClfDataSet validSet  = dataSetPair.getSecond();
-            datasets.add(subTrain);
-            datasets.add(validSet);
-        } else {
-            MultiLabelClfDataSet validSet = TRECFormat.loadMultiLabelClfDataSetAutoSparseSequential(config.getString("input.validData"));
-            datasets.add(trainSet);
-            datasets.add(validSet);
-        }
-        return datasets;
+    private static MultiLabelClfDataSet loadValidData(Config config) throws Exception{
+        MultiLabelClfDataSet validSet = TRECFormat.loadMultiLabelClfDataSetAutoSparseSequential(config.getString("input.validData"));
+        return validSet;
     }
 
     private static MultiLabelClfDataSet loadTrainData(Config config) throws Exception{
@@ -525,7 +321,6 @@ public class BRLREN {
     private static class HyperParameters{
         double penalty;
         double l1Ratio;
-        int iterations;
         int numComponents;
 
         HyperParameters() {
@@ -534,7 +329,6 @@ public class BRLREN {
         HyperParameters(Config config) {
             penalty = config.getDouble("train.penalty");
             l1Ratio = config.getDouble("train.l1Ratio");
-            iterations = config.getInt("train.iterations");
             numComponents = config.getInt("train.numComponents");
         }
 
@@ -542,7 +336,6 @@ public class BRLREN {
             Config config = new Config();
             config.setDouble("train.penalty", penalty);
             config.setDouble("train.l1Ratio", l1Ratio);
-            config.setInt("train.iterations", iterations);
             config.setInt("train.numComponents", numComponents);
             return config;
         }
@@ -550,10 +343,6 @@ public class BRLREN {
 
     }
 
-    private static class TuneResult{
-        HyperParameters hyperParameters;
-        double performance;
-    }
 
     private static boolean containsNovelClass(MultiLabel multiLabel, List<Integer> novelLabels){
         for (int l:novelLabels){

early stopping