@Test
public void shouldCreateTokensForWordsWithAlphabeticCharacters() {

    String content = "This is a series of words.";
    tokenize(content);
    assertNextTokenIs(0, 4, CndTokenizer.WORD);
    assertNextTokenIs(5, 7, CndTokenizer.WORD);
    assertNextTokenIs(8, 9, CndTokenizer.WORD);
    assertNextTokenIs(10, 16, CndTokenizer.WORD);
    assertNextTokenIs(17, 19, CndTokenizer.WORD);
    assertNextTokenIs(20, 26, CndTokenizer.WORD);
    assertNoMoreTokens();
}