Examples of org.antlr.v4.runtime.CommonTokenStream

org.antlr.v4.runtime.CommonTokenStream
This class extends {@link BufferedTokenStream} with functionality to filtertoken streams to tokens on a particular channel (tokens where {@link Token#getChannel} returns a particular value).
This token stream provides access to all tokens by index or when calling methods like {@link #getText}. The channel filtering is only used for code accessing tokens via the lookahead methods {@link #LA}, {@link #LT}, and {@link #LB}.

By default, tokens are placed on the default channel ( {@link Token#DEFAULT_CHANNEL}), but may be reassigned by using the {@code ->channel(HIDDEN)} lexer command, or by using an embedded action tocall {@link Lexer#setChannel}.

Note: lexer rules which use the {@code ->skip} lexer command or call{@link Lexer#skip} do not produce tokens at all, so input text matched bysuch a rule will not be available as part of the token stream, regardless of channel.

                       "A : 'a';\n" +
                       "B : 'b';\n" +
                       "C : 'c';\n");
    String input = "abcccba";
    LexerInterpreter lexEngine = g.createLexerInterpreter(new ANTLRInputStream(input));
    CommonTokenStream stream = new CommonTokenStream(lexEngine);
    stream.fill();
    TokenStreamRewriter tokens = new TokenStreamRewriter(stream);
    tokens.replace(2, 4, "x");
    tokens.insertBefore(4, "y");
    stream.fill(); // no effect; within range of a replace
    Exception exc = null;
    try {
      tokens.getText();
    }
    catch (IllegalArgumentException iae) {

View Full Code Here

                       "A : 'a';\n" +
                       "B : 'b';\n" +
                       "C : 'c';\n");
    String input = "abcccba";
    LexerInterpreter lexEngine = g.createLexerInterpreter(new ANTLRInputStream(input));
    CommonTokenStream stream = new CommonTokenStream(lexEngine);
    stream.fill();
    TokenStreamRewriter tokens = new TokenStreamRewriter(stream);
    tokens.replace(2, 4, "x");
    tokens.insertAfter(4, "y");
    String result = tokens.getText();
    String expecting = "abxyba";

View Full Code Here

                       "A : 'a';\n" +
                       "B : 'b';\n" +
                       "C : 'c';\n");
    String input = "abcccba";
    LexerInterpreter lexEngine = g.createLexerInterpreter(new ANTLRInputStream(input));
    CommonTokenStream stream = new CommonTokenStream(lexEngine);
    stream.fill();
    TokenStreamRewriter tokens = new TokenStreamRewriter(stream);
    tokens.replace(0, 6, "x");
    String result = tokens.getText();
    String expecting = "x";
    assertEquals(expecting, result);

View Full Code Here

                       "A : 'a';\n" +
                       "B : 'b';\n" +
                       "C : 'c';\n");
    String input = "abcccba";
    LexerInterpreter lexEngine = g.createLexerInterpreter(new ANTLRInputStream(input));
    CommonTokenStream stream = new CommonTokenStream(lexEngine);
    stream.fill();
    TokenStreamRewriter tokens = new TokenStreamRewriter(stream);
    tokens.replace(2, 4, "xyz");
    String result = tokens.getText(Interval.of(0, 6));
    String expecting = "abxyzba";
    assertEquals(expecting, result);

View Full Code Here

                       "A : 'a';\n" +
                       "B : 'b';\n" +
                       "C : 'c';\n");
    String input = "abcccba";
    LexerInterpreter lexEngine = g.createLexerInterpreter(new ANTLRInputStream(input));
    CommonTokenStream stream = new CommonTokenStream(lexEngine);
    stream.fill();
    TokenStreamRewriter tokens = new TokenStreamRewriter(stream);
    tokens.replace(2, 4, "xyz");
    tokens.replace(3, 5, "foo");
    stream.fill();
// overlaps, error
    Exception exc = null;
    try {
      tokens.getText();
    }

View Full Code Here

                       "A : 'a';\n" +
                       "B : 'b';\n" +
                       "C : 'c';\n");
    String input = "abcccba";
    LexerInterpreter lexEngine = g.createLexerInterpreter(new ANTLRInputStream(input));
    CommonTokenStream stream = new CommonTokenStream(lexEngine);
    stream.fill();
    TokenStreamRewriter tokens = new TokenStreamRewriter(stream);
    tokens.replace(2, 4, "xyz");
    tokens.replace(1, 3, "foo");
    stream.fill();
// overlap, error
    Exception exc = null;
    try {
      tokens.getText();
    }

View Full Code Here

                       "A : 'a';\n" +
                       "B : 'b';\n" +
                       "C : 'c';\n");
    String input = "abcba";
    LexerInterpreter lexEngine = g.createLexerInterpreter(new ANTLRInputStream(input));
    CommonTokenStream stream = new CommonTokenStream(lexEngine);
    stream.fill();
    TokenStreamRewriter tokens = new TokenStreamRewriter(stream);
    tokens.replace(2, 2, "xyz");
    tokens.replace(0, 3, "foo");
    String result = tokens.getText();
    String expecting = "fooa";

View Full Code Here

                       "A : 'a';\n" +
                       "B : 'b';\n" +
                       "C : 'c';\n");
    String input = "abc";
    LexerInterpreter lexEngine = g.createLexerInterpreter(new ANTLRInputStream(input));
    CommonTokenStream stream = new CommonTokenStream(lexEngine);
    stream.fill();
    TokenStreamRewriter tokens = new TokenStreamRewriter(stream);
    tokens.insertBefore(0, "x");
    tokens.insertBefore(0, "y");
    String result = tokens.getText();
    String expecting = "yxabc";

View Full Code Here

                       "A : 'a';\n" +
                       "B : 'b';\n" +
                       "C : 'c';\n");
    String input = "abc";
    LexerInterpreter lexEngine = g.createLexerInterpreter(new ANTLRInputStream(input));
    CommonTokenStream stream = new CommonTokenStream(lexEngine);
    stream.fill();
    TokenStreamRewriter tokens = new TokenStreamRewriter(stream);
    tokens.insertBefore(1, "x");
    tokens.insertBefore(0, "y");
    tokens.insertBefore(1, "z");
    String result = tokens.getText();

View Full Code Here

                       "A : 'a';\n" +
                       "B : 'b';\n" +
                       "C : 'c';\n");
    String input = "abc";
    LexerInterpreter lexEngine = g.createLexerInterpreter(new ANTLRInputStream(input));
    CommonTokenStream stream = new CommonTokenStream(lexEngine);
    stream.fill();
    TokenStreamRewriter tokens = new TokenStreamRewriter(stream);
    tokens.replace(0, 2, "foo");
    tokens.insertBefore(0, "z");
    stream.fill();
// combine with left edge of rewrite
    String result = tokens.getText();
    String expecting = "zfoo";
    assertEquals(expecting, result);
  }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.antlr.v4.runtime.CommonTokenStream

com.espertech.esper.core.deploy.EPLModuleUtil

com.espertech.esper.event.property.PropertyParser

org.antlr.runtime.ANTLRStringStream

org.antlr.runtime.CharStream

org.antlr.runtime.CommonToken

org.antlr.runtime.Token

org.antlr.runtime.tree.CommonTree

org.antlr.runtime.tree.CommonTreeNodeStream

org.antlr.runtime.tree.Tree

org.antlr.runtime.tree.TreeVisitor

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.