class TextDelimitedStringTokenizer implements StringTokenizer {

    @Override
    public List<String> tokenize(String input)
    {
        Utilities.checkNull(input, "input");

        // Substitute a single non-printable character for multi-character
        // delimiters to simplify processing
        String replaced = substituteDelimiters(input);

        // Reduce double open and close delimiters
        String reduced = reduceTextDelimiters(replaced);

        char open = ourFinalOpen;
        char close = myUsesSingleTextDelimiter ? ourFinalOpen : ourFinalClose;
        List<String> tokens = new ArrayList<>();
        int tokenStart = 0;
        boolean inTextDelim = false;
        for (int i = 0; i < reduced.length(); ++i)
        {
            char current = reduced.charAt(i);
            if (inTextDelim)
            {
                if (current == close)
                {
                    inTextDelim = false;
                }
            }
            else if (current == open)
            {
                inTextDelim = true;
            }
            else if (current == ourSingleCharTokenDelimiter)
            {
                if (tokenStart != -1)
                {
                    tokens.add(postProcessToken(reduced.substring(tokenStart, i)));
                }
                tokenStart = i + 1;
            }
        }

        if (tokenStart != -1)
        {
            tokens.add(postProcessToken(reduced.substring(tokenStart, reduced.length())));
        }

        return tokens;
    }

    public  TextDelimitedStringTokenizer(String tokenDelimiter);
    public  TextDelimitedStringTokenizer(String tokenDelimiter, String textDelimiter);
    public  TextDelimitedStringTokenizer(String tokenDelimiter, String textDelimiterOpen, String textDelimiterClose);

}