<NME> Exp69.java
<BEF> ADDFILE
<MSG> add exp69

<DFF> @@ -0,0 +1,182 @@
+package edu.neu.ccs.pyramid.experiment;
+
+import edu.neu.ccs.pyramid.configuration.Config;
+import edu.neu.ccs.pyramid.dataset.*;
+import edu.neu.ccs.pyramid.elasticsearch.ESIndex;
+import edu.neu.ccs.pyramid.elasticsearch.SingleLabelIndex;
+import edu.neu.ccs.pyramid.elasticsearch.TermStat;
+import edu.neu.ccs.pyramid.feature.FeatureMappers;
+import edu.neu.ccs.pyramid.feature.NumericalFeatureMapper;
+import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.search.SearchHit;
+
+import java.io.IOException;
+import java.util.*;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+
+/**
+ * check white house vs white/house
+ * Created by chengli on 2/25/15.
+ */
+public class Exp69 {
+    public static void main(String[] args) {
+
+    }
+
+    static SingleLabelIndex loadIndex(Config config) throws Exception{
+        SingleLabelIndex.Builder builder = new SingleLabelIndex.Builder()
+                .setIndexName(config.getString("index.indexName"))
+                .setClusterName(config.getString("index.clusterName"))
+                .setClientType(config.getString("index.clientType"))
+                .setLabelField(config.getString("index.labelField"))
+                .setExtLabelField(config.getString("index.extLabelField"))
+                .setDocumentType(config.getString("index.documentType"));
+        if (config.getString("index.clientType").equals("transport")){
+            String[] hosts = config.getString("index.hosts").split(Pattern.quote(","));
+            String[] ports = config.getString("index.ports").split(Pattern.quote(","));
+            builder.addHostsAndPorts(hosts,ports);
+        }
+        SingleLabelIndex index = builder.build();
+        System.out.println("index loaded");
+        System.out.println("there are "+index.getNumDocs()+" documents in the index.");
+//        for (int i=0;i<index.getNumDocs();i++){
+//            System.out.println(i);
+//            System.out.println(index.getLabel(""+i));
+//        }
+        return index;
+    }
+
+    static ClfDataSet loadTrainSet(Config config, SingleLabelIndex index, FeatureMappers featureMappers,
+                                   IdTranslator idTranslator, LabelTranslator labelTranslator) throws Exception{
+        System.out.println("creating training set");
+        int totalDim = config.getInt("featureMatrix.maxNumColumns");
+        System.out.println("allocating "+totalDim+" columns for training set");
+        ClfDataSet dataSet = loadData(config,index,featureMappers,idTranslator,totalDim,labelTranslator);
+        System.out.println("training set created");
+        return dataSet;
+    }
+
+    static ClfDataSet loadData(Config config, SingleLabelIndex index,
+                               FeatureMappers featureMappers,
+                               IdTranslator idTranslator, int totalDim,
+                               LabelTranslator labelTranslator) throws Exception{
+        int numDataPoints = idTranslator.numData();
+        int numClasses = config.getInt("numClasses");
+        ClfDataSet dataSet = ClfDataSetBuilder.getBuilder()
+                .numDataPoints(numDataPoints).numFeatures(totalDim)
+                .numClasses(numClasses).dense(!config.getBoolean("featureMatrix.sparse"))
+                .missingValue(config.getBoolean("featureMatrix.missingValue"))
+                .build();
+
+        IntStream.range(0, numDataPoints).parallel()
+                .forEach(i -> {
+                    String dataIndexId = idTranslator.toExtId(i);
+                    int label = index.getLabel(dataIndexId);
+                    dataSet.setLabel(i,label);
+                });
+
+        String[] dataIndexIds = idTranslator.getAllExtIds();
+
+        featureMappers.getCategoricalFeatureMappers().stream().parallel().
+                forEach(categoricalFeatureMapper -> {
+                    String featureName = categoricalFeatureMapper.getFeatureName();
+                    String source = categoricalFeatureMapper.getSource();
+                    if (source.equalsIgnoreCase("field")){
+                        for (String id: dataIndexIds){
+                            int algorithmId = idTranslator.toIntId(id);
+                            String category = index.getStringField(id,featureName);
+                            // might be a new category unseen in training
+                            if (categoricalFeatureMapper.hasCategory(category)){
+                                int featureIndex = categoricalFeatureMapper.getFeatureIndex(category);
+                                dataSet.setFeatureValue(algorithmId,featureIndex,1);
+                            }
+                        }
+                    }
+                });
+
+
+        featureMappers.getNumericalFeatureMappers().stream().parallel().
+                forEach(numericalFeatureMapper -> {
+                    String featureName = numericalFeatureMapper.getFeatureName();
+                    String source = numericalFeatureMapper.getSource();
+                    int featureIndex = numericalFeatureMapper.getFeatureIndex();
+
+                    if (source.equalsIgnoreCase("field")){
+                        for (String id: dataIndexIds){
+                            int algorithmId = idTranslator.toIntId(id);
+                            float value = index.getFloatField(id,featureName);
+                            dataSet.setFeatureValue(algorithmId,featureIndex,value);
+                        }
+                    }
+
+                    if (source.equalsIgnoreCase("matching_score")){
+                        SearchResponse response = null;
+
+                        //todo assume unigram, so slop doesn't matter
+                        response = index.matchPhrase(index.getBodyField(), featureName, dataIndexIds, 0);
+
+                        SearchHit[] hits = response.getHits().getHits();
+                        for (SearchHit hit: hits){
+                            String indexId = hit.getId();
+                            float score = hit.getScore();
+                            int algorithmId = idTranslator.toIntId(indexId);
+                            dataSet.setFeatureValue(algorithmId,featureIndex,score);
+                        }
+                    }
+                });
+        DataSetUtil.setIdTranslator(dataSet, idTranslator);
+        DataSetUtil.setLabelTranslator(dataSet, labelTranslator);
+        return dataSet;
+    }
+
+    static List<String> gatherUnigrams(ESIndex index, String field,
+                                       String[] ids, int minDf) throws Exception{
+        System.out.println("gathering unigrams with minDf "+minDf+" from field "+field);
+        Set<TermStat> unigrams = Collections.newSetFromMap(new ConcurrentHashMap<TermStat, Boolean>());
+        Arrays.stream(ids).parallel().forEach(id -> {
+            Set<TermStat> termStats = null;
+            try {
+                termStats = index.getTermStats(field, id);
+            } catch (IOException e) {
+                System.out.println("id= "+id);
+                e.printStackTrace();
+            }
+            termStats.stream().filter(termStat -> termStat.getDf() > minDf).forEach(unigrams::add);
+        });
+
+        List<String> list = unigrams.stream().sorted(Comparator.comparing(TermStat::getTerm))
+                .sorted(Comparator.comparing(TermStat::getDf).reversed())
+                .map(TermStat::getTerm)
+                .collect(Collectors.toList());
+        System.out.println("done");
+        System.out.println("there are "+list.size()+" unigrams");
+        return list;
+    }
+
+    static String[] sampleTrain(Config config, SingleLabelIndex index, Set<String> duplicate){
+        int numDocsInIndex = index.getNumDocs();
+        String[] ids = null;
+
+        String splitField = config.getString("index.splitField");
+        List<String> train = IntStream.range(0, numDocsInIndex).parallel()
+                .filter(i -> index.getStringField("" + i, splitField).
+                        equalsIgnoreCase("train")).
+                        mapToObj(i -> "" + i).filter(id -> !duplicate.contains(id)).collect(Collectors.toList());
+        ids = train.toArray(new String[train.size()]);
+        return ids;
+    }
+
+    static void addUnigrams(FeatureMappers featureMappers, String[] unigrams){
+        for (String unigram: unigrams){
+            int featureIndex = featureMappers.nextAvailable();
+            NumericalFeatureMapper mapper = NumericalFeatureMapper.getBuilder().
+                    setFeatureIndex(featureIndex).setFeatureName(unigram)
+                    .build();
+            mapper.getSettings().put("source","matching_score");
+            featureMappers.addMapper(mapper);
+        }
+    }
+}

add exp69