class TokenStream {

  List<Token> tokenize() {
    List<Token> tokenList = new ArrayList<Token>();

    while (true) {
      String currentLine;
      try {
        currentLine = lineReader.readLine();
      } catch (IOException e) {
        throw new MessageConveyorException("Failed to read input stream", e);
      }
      if (currentLine == null) {
        break;
      }
      if(state != State.TRAILING_BACKSLASH) {
        state = State.START;
      }
      tokenizeLine(tokenList, currentLine);
      tokenList.add(Token.EOL);
    }

    return tokenList;
  }

    TokenStream(Reader reader);

}