@Test
public void tokenize(){

	Pattern pattern = Pattern.compile("\\s+");
	TextTokenizer tokenizer = new TextTokenizer(pattern);
	assertEquals(Collections.emptyList(), tokenizer.tokenize(""));
	assertEquals(Collections.emptyList(), tokenizer.tokenize(" "));
	assertEquals(Collections.emptyList(), tokenizer.tokenize("\t\t\t"));
	assertEquals(Collections.emptyList(), tokenizer.tokenize(","));
	assertEquals(Collections.emptyList(), tokenizer.tokenize(",,"));
	assertEquals(Collections.emptyList(), tokenizer.tokenize(", ,"));
	assertEquals(Collections.emptyList(), tokenizer.tokenize(" , , "));
	assertEquals(Arrays.asList("one", "two", "three"), tokenizer.tokenize("one two three"));
	assertEquals(Arrays.asList("one", "two", "three"), tokenizer.tokenize("one!, \u00BFtwo?, three."));
}