<NME> FeatureLoader.java
<BEF> package edu.neu.ccs.pyramid.elasticsearch;

import edu.neu.ccs.pyramid.dataset.*;
import edu.neu.ccs.pyramid.feature.*;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.search.SearchHit;

import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.stream.IntStream;

/**
 * Created by chengli on 3/5/15.
 */
public class FeatureLoader {


    public static void loadFeatures(ESIndex index, DataSet dataSet, FeatureList features,
                                    IdTranslator idTranslator, MatchScoreType matchScoreType, String docFilter,
                                    Map<String, float[]> fieldLength){
//        ProgressBar progressBar = new ProgressBar(features.size());
        IntStream.range(0,features.size())
        		.parallel()
                .forEach(i-> {
                    Feature feature = features.get(i);
                    if (feature instanceof CategoricalFeature){
                        loadCategoricalFeature(index,dataSet,(CategoricalFeature)feature,idTranslator, docFilter);
                    } else if (feature instanceof Ngram){
                        loadNgramFeature(index, dataSet, (Ngram)feature, idTranslator, matchScoreType, docFilter, fieldLength);
                    } else if (feature instanceof CodeDescription) {
                        loadCodeDesFeatureTfidf(index, dataSet, feature, idTranslator, docFilter);
//                        loadCodeDesFeature(index, dataSet, feature, idTranslator, docFilter);
                    } else {
                        loadNumericalFeature(index,dataSet,feature,idTranslator);
                    }

//                    progressBar.incrementAndPrint();
                }
                );
//        System.out.println();
    }

    public static void loadCategoricalFeature(ESIndex index, DataSet dataSet, CategoricalFeature feature,

    public static void loadCategoricalFeature(ESIndex index, DataSet dataSet, CategoricalFeature feature,
                                              IdTranslator idTranslator, String docFilter){
        String[] dataIndexIds = idTranslator.getAllExtIds();
        String variableName = feature.getVariableName();
        int featureIndex = feature.getIndex();

            matchedIds = index.termFilter(variableName,feature.getCategory(),docFilter,idTranslator.numData());
        } catch (Exception e) {
            e.printStackTrace();
        }
        for (String matchedId: matchedIds){
            int algorithmId = idTranslator.toIntId(matchedId);
            dataSet.setFeatureValue(algorithmId,featureIndex,1);
        }
            dataSet.setFeatureValue(algorithmId,featureIndex,1);
        }

        List<String> docMissingField = index.docsWithFieldMissing(variableName,dataIndexIds);
        for (String extId: docMissingField){
            int algorithmId = idTranslator.toIntId(extId);
            dataSet.setFeatureValue(algorithmId,featureIndex,Double.NaN);
        }


    }
                                        IdTranslator idTranslator, MatchScoreType matchScoreType, String docFilter,
                                        Map<String, float[]> fieldLength){
        switch (matchScoreType){
            case ES_ORIGINAL:
                loadNgramFeatureOriginal(index, dataSet, feature, idTranslator, docFilter);
                break;
            case BINARY:
                loadNgramFeatureBinary(index, dataSet, feature, idTranslator, docFilter);
                break;
            case FREQUENCY:
                loadNgramFeatureFrequency(index, dataSet, feature, idTranslator, docFilter);
                break;
            case TFIFL:
                loadNgramFeatureTFIFL(index, dataSet, feature, idTranslator, docFilter, fieldLength);
        }
    }


    private static void loadNgramFeatureOriginal(ESIndex index, DataSet dataSet, Ngram feature,
                                                 IdTranslator idTranslator, String docFilter){
        int featureIndex = feature.getIndex();
        SearchResponse response = index.spanNear(feature, docFilter, idTranslator.numData());
        SearchHit[] hits = response.getHits().getHits();
        for (SearchHit hit: hits){
            String indexId = hit.getId();
            float score = hit.getScore();
            int algorithmId = idTranslator.toIntId(indexId);
            dataSet.setFeatureValue(algorithmId,featureIndex,score);
        }
    }

    private static void loadNgramFeatureFrequency(ESIndex index, DataSet dataSet, Ngram feature,
                                                  IdTranslator idTranslator, String docFilter){
        int featureIndex = feature.getIndex();
        SearchResponse response = index.spanNearFrequency(feature, docFilter, idTranslator.numData());
        SearchHit[] hits = response.getHits().getHits();
        for (SearchHit hit: hits){
            String indexId = hit.getId();
            float score = hit.getScore();
            int algorithmId = idTranslator.toIntId(indexId);
            dataSet.setFeatureValue(algorithmId,featureIndex,score);
        }
    }

    // term frequency inverse field length
    // field storing the length of the body field should be called body_field_length
    // todo avoid repeated retrieval of the same field length
    private static void loadNgramFeatureTFIFL(ESIndex index, DataSet dataSet, Ngram feature,
                                              IdTranslator idTranslator, String docFilter,
                                              Map<String, float[]> fieldLength){
        int featureIndex = feature.getIndex();
        SearchResponse response = index.spanNearFrequency(feature, docFilter, idTranslator.numData());
        SearchHit[] hits = response.getHits().getHits();
        String field = feature.getField();
        for (SearchHit hit: hits){
            String indexId = hit.getId();
            float score = hit.getScore();
            int algorithmId = idTranslator.toIntId(indexId);
            float docLength = fieldLength.get(field)[algorithmId];
            double s = score/docLength;

            dataSet.setFeatureValue(algorithmId,featureIndex,s);
        }
    }

    public static void loadNgramFeatureBinary(ESIndex index, DataSet dataSet, Ngram feature,
                                              IdTranslator idTranslator, String docFilter){
        int featureIndex = feature.getIndex();
        SearchResponse response = index.spanNear(feature, docFilter, idTranslator.numData());
        SearchHit[] hits = response.getHits().getHits();
        for (SearchHit hit: hits){
            String indexId = hit.getId();
            float score = hit.getScore();
            int algorithmId = idTranslator.toIntId(indexId);
            if (score>0){
                score=1;
            }
            dataSet.setFeatureValue(algorithmId,featureIndex,score);
        }
    }

//    public static void loadSpanNotNgramFeature(ESIndex index, DataSet dataSet, SpanNotNgram feature,
//                                        IdTranslator idTranslator){
//        String[] dataIndexIds = idTranslator.getAllExtIds();
//
//        int featureIndex = feature.getIndex();
//
//        SearchResponse response = index.spanNot(feature, dataIndexIds);
//        SearchHit[] hits = response.getHits().getHits();
//        for (SearchHit hit: hits){
//            String indexId = hit.getId();
//            float score = hit.getScore();
//            int algorithmId = idTranslator.toIntId(indexId);
//            dataSet.setFeatureValue(algorithmId,featureIndex,score);
//        }
//    }

//    public static Vector loadNgramFeature(ESIndex index, Ngram feature, IdTranslator idTranslator ){
//        String[] dataIndexIds = idTranslator.getAllExtIds();
//        SearchResponse response = index.spanNear(feature, dataIndexIds);
//        SearchHit[] hits = response.getHits().getHits();
//        Vector vector = new RandomAccessSparseVector(idTranslator.numData());
//        for (SearchHit hit: hits){
//            String indexId = hit.getId();
//            float score = hit.getScore();
//            int algorithmId = idTranslator.toIntId(indexId);
//            vector.set(algorithmId,score);
//        }
//        return vector;
//    }


    public static void loadNumericalFeature(ESIndex index, DataSet dataSet, Feature feature,
                                              IdTranslator idTranslator){
        String[] dataIndexIds = idTranslator.getAllExtIds();
        String variableName = feature.getName();
        int featureIndex = feature.getIndex();
        String source = feature.getSettings().get("source");
        if (source.equals("field")){
            Arrays.stream(dataIndexIds).forEach(id -> {
                int algorithmId = idTranslator.toIntId(id);
                //may return NaN
                double value = index.getFloatField(id,variableName);
                dataSet.setFeatureValue(algorithmId, featureIndex, value);
            });
        }
    }


    private static void loadCodeDesFeature(ESIndex index, DataSet dataSet, Feature feature,
                                           IdTranslator idTranslator, String docFilter){

        int featureIndex = feature.getIndex();
        CodeDescription codeDescription = (CodeDescription)(feature);
        SearchResponse response = index.minimumShouldMatch(codeDescription.getDescription(), codeDescription.getField(), codeDescription.getPercentage(), idTranslator.numData(), docFilter);
        SearchHit[] hits = response.getHits().getHits();
        for (SearchHit hit: hits){
            String indexId = hit.getId();
            float score = hit.getScore();
            int algorithmId = idTranslator.toIntId(indexId);
            dataSet.setFeatureValue(algorithmId,featureIndex,score);
        }

    }


    private static void loadCodeDesFeatureTfidf(ESIndex index, DataSet dataSet, Feature feature,
                                                IdTranslator idTranslator, String docFilter){
        int featureIndex = feature.getIndex();
        CodeDescription codeDescription = (CodeDescription)(feature);

        SearchResponse response = index.match(codeDescription.getField(),codeDescription.getDescriptionString(),
                    docFilter,
                    codeDescription.getSize());
        SearchHit[] hits = response.getHits().getHits();
        for (SearchHit hit: hits){
            String indexId = hit.getId();
            float score = hit.getScore();
            int algorithmId = idTranslator.toIntId(indexId);
            dataSet.setFeatureValue(algorithmId,featureIndex,score);
        }

    }














    public static enum MatchScoreType{
        ES_ORIGINAL, BINARY, FREQUENCY, TFIFL
    }
}

<MSG> fix slowness in categorical feature matching

<DFF> @@ -45,7 +45,7 @@ public class FeatureLoader {
 
     public static void loadCategoricalFeature(ESIndex index, DataSet dataSet, CategoricalFeature feature,
                                               IdTranslator idTranslator, String docFilter){
-        String[] dataIndexIds = idTranslator.getAllExtIds();
+//        String[] dataIndexIds = idTranslator.getAllExtIds();
         String variableName = feature.getVariableName();
         int featureIndex = feature.getIndex();
 
@@ -60,11 +60,11 @@ public class FeatureLoader {
             dataSet.setFeatureValue(algorithmId,featureIndex,1);
         }
 
-        List<String> docMissingField = index.docsWithFieldMissing(variableName,dataIndexIds);
-        for (String extId: docMissingField){
-            int algorithmId = idTranslator.toIntId(extId);
-            dataSet.setFeatureValue(algorithmId,featureIndex,Double.NaN);
-        }
+//        List<String> docMissingField = index.docsWithFieldMissing(variableName,dataIndexIds);
+//        for (String extId: docMissingField){
+//            int algorithmId = idTranslator.toIntId(extId);
+//            dataSet.setFeatureValue(algorithmId,featureIndex,Double.NaN);
+//        }
 
 
     }

fix slowness in categorical feature matching