List<Token> tokenize() throws ScanException {
        List<Token> tokenList = new ArrayList<Token>();
        StringBuffer buf = new StringBuffer();

        while (pointer < patternLength) {
            char c = pattern.charAt(pointer);
            pointer++;

            switch (state) {
            case LITERAL_STATE:
                handleLiteralState(c, tokenList, buf);
                break;
            case FORMAT_MODIFIER_STATE:
                handleFormatModifierState(c, tokenList, buf);
                break;
            case OPTION_STATE:
                processOption(c, tokenList, buf);
                break;
            case KEYWORD_STATE:
                handleKeywordState(c, tokenList, buf);
                break;
            case RIGHT_PARENTHESIS_STATE:
                handleRightParenthesisState(c, tokenList, buf);
                break;

            default:
            }
        }

        // EOS
        switch (state) {
        case LITERAL_STATE:
            addValuedToken(Token.LITERAL, buf, tokenList);
            break;
        case KEYWORD_STATE:
            tokenList.add(new Token(Token.SIMPLE_KEYWORD, buf.toString()));
            break;
        case RIGHT_PARENTHESIS_STATE:
            tokenList.add(Token.RIGHT_PARENTHESIS_TOKEN);
            break;

        case FORMAT_MODIFIER_STATE:
        case OPTION_STATE:
            throw new ScanException("Unexpected end of pattern string");
        }

        return tokenList;
    }
@Test
    public void testEmptyP2() throws ScanException {
        List<Token> tl = new TokenStream("%()").tokenize();
        List<Token> witness = new ArrayList<Token>();
        witness.add(Token.PERCENT_TOKEN);
        witness.add(Token.BARE_COMPOSITE_KEYWORD_TOKEN);
        witness.add(Token.RIGHT_PARENTHESIS_TOKEN);
        assertEquals(witness, tl);
    }