Examples of BaseJavaCCParserImplTokenManager

Examples of com.pogofish.jadt.parser.javacc.generated.BaseJavaCCParserImplTokenManager

37
38
39
40
41
42
43
44
    /**
     * Create a tokenizer that will read from the given string
     */
    private BaseJavaCCParserImplTokenManager tokenizer(String testString) {
        final Source source = new StringSource("TokenizerTest", testString);
        return new BaseJavaCCParserImplTokenManager(new JavaCharStream(
                source.createReader()));
    }
View Full Code Here

Examples of com.pogofish.jadt.parser.javacc.generated.BaseJavaCCParserImplTokenManager

46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
    /**
     * Comments should be invisible in the output other than separating tokens
     */
    @Test
    public void testComments() {
        final BaseJavaCCParserImplTokenManager tokenizer1 = tokenizer("/*\nCopyright*/hello//comment\nworld/**another comment*/oh");
        check(tokenizer1, "hello", IDENTIFIER, 2);
        check(tokenizer1, "world", IDENTIFIER, 3);
        check(tokenizer1, "oh", IDENTIFIER, 3);
        check(tokenizer1, "<EOF>", EOF, 3);

        final BaseJavaCCParserImplTokenManager tokenizer2 = tokenizer("/**/hello");
        check(tokenizer2, "hello", IDENTIFIER, 1);
        check(tokenizer2, "<EOF>", EOF, 1);

        final BaseJavaCCParserImplTokenManager tokenizer3 = tokenizer("/***/hello");
        check(tokenizer3, "hello", IDENTIFIER, 1);
        check(tokenizer3, "<EOF>", EOF, 1);

        final BaseJavaCCParserImplTokenManager tokenizer4 = tokenizer("/****/hello");
        check(tokenizer4, "hello", IDENTIFIER, 1);
        check(tokenizer4, "<EOF>", EOF, 1);

        final BaseJavaCCParserImplTokenManager tokenizer5 = tokenizer("/*** */hello");
        check(tokenizer5, "hello", IDENTIFIER, 1);
        check(tokenizer5, "<EOF>", EOF, 1);

        final BaseJavaCCParserImplTokenManager tokenizer6 = tokenizer("/* ***/hello");
        check(tokenizer6, "hello", IDENTIFIER, 1);
        check(tokenizer6, "<EOF>", EOF, 1);

        final BaseJavaCCParserImplTokenManager tokenizer7 = tokenizer("/* **/hello");
        check(tokenizer7, "hello", IDENTIFIER, 1);
        check(tokenizer7, "<EOF>", EOF, 1);

    }
View Full Code Here

Examples of com.pogofish.jadt.parser.javacc.generated.BaseJavaCCParserImplTokenManager

83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
    /**
     * Even unterminated comments should "work"
     */
    @Test
    public void testUnterminatedComments() {
        final BaseJavaCCParserImplTokenManager tokenizer1 = tokenizer("/** haha");
        check(tokenizer1, "/** haha", UNTERMINATED_COMMENT, 1);
        check(tokenizer1, "<EOF>", EOF, 1);

        final BaseJavaCCParserImplTokenManager tokenizer2 = tokenizer("/* haha");
        check(tokenizer2, "/* haha", UNTERMINATED_COMMENT, 1);
        check(tokenizer2, "<EOF>", EOF, 1);

        // I guess end of line comment ending with an EOF instead of an EOL isn't allt that unterminated
        final BaseJavaCCParserImplTokenManager tokenizer3 = tokenizer("// haha");
        check(tokenizer3, "<EOF>", EOF, 1);
    }
View Full Code Here

Examples of com.pogofish.jadt.parser.javacc.generated.BaseJavaCCParserImplTokenManager

101
102
103
104
105
106
107
108
109
110
111
    /**
     * Whitespace should be invisible in the output other than separating tokens
     */
    @Test
    public void testWhitespace() {
        final BaseJavaCCParserImplTokenManager tokenizer = tokenizer("hello    world   \toh");
        check(tokenizer, "hello", IDENTIFIER, 1);
        check(tokenizer, "world", IDENTIFIER, 1);
        check(tokenizer, "oh", IDENTIFIER, 1);
        check(tokenizer, "<EOF>", EOF, 1);
    }
View Full Code Here

Examples of com.pogofish.jadt.parser.javacc.generated.BaseJavaCCParserImplTokenManager

114
115
116
117
118
119
120
121
122
123
124
     * End of line should be invisible in the output other than separating
     * tokens and incrementing the line number
     */
    @Test
    public void testEol() {
        final BaseJavaCCParserImplTokenManager tokenizer = tokenizer("hello\nworld\ryeah\r\noh");
        check(tokenizer, "hello", IDENTIFIER, 1);
        check(tokenizer, "world", IDENTIFIER, 2);
        check(tokenizer, "yeah", IDENTIFIER, 3);
        check(tokenizer, "oh", IDENTIFIER, 4);
        check(tokenizer, "<EOF>", EOF, 4);
View Full Code Here

Examples of com.pogofish.jadt.parser.javacc.generated.BaseJavaCCParserImplTokenManager

127
128
129
130
131
132
133
134
135
136
    /**
     * Various classes of non-keyword identifiers
     */
    @Test
    public void testIdentifiers() {
        final BaseJavaCCParserImplTokenManager tokenizer = tokenizer("hello \u00a5123\u00a512342");
        check(tokenizer, "hello", IDENTIFIER, 1);
        check(tokenizer, "\u00a5123\u00a512342", IDENTIFIER, 1);
        check(tokenizer, "<EOF>", EOF, 1);
    }
View Full Code Here

Examples of com.pogofish.jadt.parser.javacc.generated.BaseJavaCCParserImplTokenManager

138
139
140
141
142
143
144
145
146
147
148
    /**
     * Test various kinds of punctuation
     */
    @Test
    public void testPunctuation() {
        final BaseJavaCCParserImplTokenManager tokenizer = tokenizer(">><<>>>>=<=<>*/+-=|(),[].@?:");
        check(tokenizer, ">>", RIGHT_SHIFT, 1);
        check(tokenizer, "<<", LEFT_SHIFT, 1);
        check(tokenizer, ">>>", ZERO_EXTENDED_RIGHT_SHIFT, 1);
        check(tokenizer, ">=", GREATER_THAN_EQUAL, 1);
        check(tokenizer, "<=", LESS_THAN_EQUAL, 1);       
View Full Code Here

Examples of com.pogofish.jadt.parser.javacc.generated.BaseJavaCCParserImplTokenManager

166
167
168
169
170
171
172
173
        check(tokenizer, "<EOF>", EOF, 1);
    }

    @Test
    public void testEOF() {
        final BaseJavaCCParserImplTokenManager tokenizer = tokenizer("");
        check(tokenizer, "<EOF>", EOF, 1);
    }
View Full Code Here

Examples of com.pogofish.jadt.parser.javacc.generated.BaseJavaCCParserImplTokenManager

175
176
177
178
179
180
181
182
183
184
185
    /**
     * Test all the reserved keywords
     */
    @Test
    public void testKeywords() {
        final BaseJavaCCParserImplTokenManager tokenizer = tokenizer("import package final extends implements class boolean byte double char float int long short abstract assert break case catch const continue "
                + "default do else enum finally for goto if instanceof interface native new private protected public return "
                + "static strictfp super switch synchronized this throw throws try void while");

        // keywords used by jADT
        check(tokenizer, "import", IMPORT, 1);
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.