class CharacterNGramTokenizer implements TextTokenizer {

    @Override
    public Iterator<Token> iterateTokens(String text) {
        Validate.notNull(text, "text must not be null");
        final String textForTokenization = padding ? createPadding(text) : text;
        return new AbstractIterator2<Token>() {
            private int offset = 0;
            private int length = minLength;

            @Override
            protected Token getNext() {
                for (;;) {
                    if (offset + minLength > textForTokenization.length()) {
                        return finished();
                    }
                    String nGram = textForTokenization.substring(offset, offset + length);
                    if (offset + length == textForTokenization.length() || length == maxLength) {
                        offset++;
                        length = minLength;
                    } else {
                        length++;
                    }
                    if (nGram.replace(PADDING_CHARACTER, "").length() == 0) {
                        continue; // skip tokens which only consist of padding character
                    }
                    // System.out.println("offset=" + offset + ",ngram=" + nGram);
                    return new ImmutableToken(offset, nGram);
                }
            }
        };
    }

    public  CharacterNGramTokenizer(int minLength, int maxLength);
    public  CharacterNGramTokenizer(int minLength, int maxLength, boolean padding);

}