@Test
public void testDFIDF() throws Exception {

	String command = String.format(hadoopCommand,
			stemmedTweets.getAbsolutePath(),
			outputLocation.getAbsolutePath(), "CSV",
			resultsOutputLocation.getAbsolutePath(), "DFIDF",
			"analysis.stemmed");
	String[] args = command.split(" ");
	args = (String[]) ArrayUtils.addAll(args, new String[] { "-pp", "-m PORTER_STEM" });
	HadoopTwitterTokenTool.main(args);
	HashMap<String, IndependentPair<Long, Long>> wordLineCounts = WordIndex.readWordCountLines(resultsOutputLocation.getAbsolutePath());
	assertTrue(wordLineCounts.get(".").firstObject() == 12);
	HashMap<Long, IndependentPair<Long, Long>> timeLineCounts = TimeIndex
			.readTimeCountLines(resultsOutputLocation.getAbsolutePath());
	long nenglish = 43;
	long sum = 0;
	for (IndependentPair<Long, Long> countLine : timeLineCounts.values()) {
		sum += countLine.firstObject();
	}
	assertTrue(sum == nenglish);
}