@Test
public void shouldCreateTokensForWordsWithNumericCharacters() {

    String content = "1234 4 5353.324";
    tokenize(content);
    assertNextTokenIs(0, 4, CndTokenizer.WORD);
    assertNextTokenIs(5, 6, CndTokenizer.WORD);
    assertNextTokenIs(7, 15, CndTokenizer.WORD);
    assertNoMoreTokens();
}