<NME> Exp56.java
<BEF> ADDFILE
<MSG> add Exp56

<DFF> @@ -0,0 +1,69 @@
+package edu.neu.ccs.pyramid.experiment;
+
+import edu.neu.ccs.pyramid.configuration.Config;
+import edu.neu.ccs.pyramid.elasticsearch.DuplicateDetector;
+import edu.neu.ccs.pyramid.elasticsearch.ESIndex;
+import edu.neu.ccs.pyramid.elasticsearch.SingleLabelIndex;
+
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileWriter;
+import java.util.regex.Pattern;
+
+/**
+ * detect duplicated docs in an index
+ * Created by chengli on 1/20/15.
+ */
+public class Exp56 {
+    public static void main(String[] args) throws Exception{
+        if (args.length !=1){
+            throw new IllegalArgumentException("please specify the config file");
+        }
+
+        Config config = new Config(args[0]);
+        System.out.println(config);
+
+        SingleLabelIndex index = loadIndex(config);
+        deduplicate(config,index);
+
+
+        index.close();
+
+    }
+
+    static SingleLabelIndex loadIndex(Config config) throws Exception{
+        SingleLabelIndex.Builder builder = new SingleLabelIndex.Builder()
+                .setIndexName(config.getString("index.indexName"))
+                .setClusterName(config.getString("index.clusterName"))
+                .setClientType(config.getString("index.clientType"))
+                .setLabelField(config.getString("index.labelField"))
+                .setExtLabelField(config.getString("index.extLabelField"))
+                .setDocumentType(config.getString("index.documentType"));
+        if (config.getString("index.clientType").equals("transport")){
+            String[] hosts = config.getString("index.hosts").split(Pattern.quote(","));
+            String[] ports = config.getString("index.ports").split(Pattern.quote(","));
+            builder.addHostsAndPorts(hosts,ports);
+        }
+        SingleLabelIndex index = builder.build();
+        System.out.println("index loaded");
+        System.out.println("there are "+index.getNumDocs()+" documents in the index.");
+//        for (int i=0;i<index.getNumDocs();i++){
+//            System.out.println(i);
+//            System.out.println(index.getLabel(""+i));
+//        }
+        return index;
+    }
+
+    static void deduplicate(Config config, ESIndex esIndex) throws Exception{
+        DuplicateDetector duplicateDetector = new DuplicateDetector(esIndex);
+        duplicateDetector.detect();
+        System.out.println("number of duplicates");
+        System.out.println(duplicateDetector.getAllDuplicates().size());
+        System.out.println(duplicateDetector.toString());
+        File file = new File(config.getString("archive.duplicate"));
+        file.getParentFile().mkdirs();
+        BufferedWriter bufferedWriter = new BufferedWriter(new FileWriter(file));
+        bufferedWriter.write(duplicateDetector.toString());
+        bufferedWriter.close();
+    }
+}

add Exp56