@Test
public void testTokenize() {

    System.out.println("tokenize");
    String[] cases = {
        "",
        "aSingleToken",
        "a b c",
        "an's g'ri # quotation test",
        "Shell instance = new Shell(new ShellTest(), System.out",
        "dir \"E:\\ASG\\!dynamic\\projects\" \t-l 3492.9  ",
        "a b c ''",
        " \"\" "
    };
    int[] sizes = {
        0, 1, 3, 1, 7, 4, 4, 1
    };
    for (int i = 0; i < cases.length; i++) {
        List<Token> result = Token.tokenize(cases[i]);
        System.out.println("case: " + cases[i]);
        for (Token t : result) {
            System.out.print(t + " ");
        }
        System.out.println();
        assertEquals(sizes[i], result.size());
    }
}