Examples of org.antlr.v4.runtime.CommonTokenStream

org.antlr.v4.runtime.CommonTokenStream
This class extends {@link BufferedTokenStream} with functionality to filtertoken streams to tokens on a particular channel (tokens where {@link Token#getChannel} returns a particular value).
This token stream provides access to all tokens by index or when calling methods like {@link #getText}. The channel filtering is only used for code accessing tokens via the lookahead methods {@link #LA}, {@link #LT}, and {@link #LB}.

By default, tokens are placed on the default channel ( {@link Token#DEFAULT_CHANNEL}), but may be reassigned by using the {@code ->channel(HIDDEN)} lexer command, or by using an embedded action tocall {@link Lexer#setChannel}.

Note: lexer rules which use the {@code ->skip} lexer command or call{@link Lexer#skip} do not produce tokens at all, so input text matched bysuch a rule will not be available as part of the token stream, regardless of channel.

                       "A : 'a';\n" +
                       "B : 'b';\n" +
                       "C : 'c';\n");
    String input = "abc";
    LexerInterpreter lexEngine = g.createLexerInterpreter(new ANTLRInputStream(input));
    CommonTokenStream stream = new CommonTokenStream(lexEngine);
    stream.fill();
    TokenStreamRewriter tokens = new TokenStreamRewriter(stream);
    tokens.delete(0, 2);
    tokens.insertBefore(0, "z");
    stream.fill();
// combine with left edge of rewrite
    String result = tokens.getText();
    String expecting = "z";
    stream.fill();
// make sure combo is not znull
    assertEquals(expecting, result);
  }

View Full Code Here

                       "A : 'a';\n" +
                       "B : 'b';\n" +
                       "C : 'c';\n");
    String input = "abc";
    LexerInterpreter lexEngine = g.createLexerInterpreter(new ANTLRInputStream(input));
    CommonTokenStream stream = new CommonTokenStream(lexEngine);
    stream.fill();
    TokenStreamRewriter tokens = new TokenStreamRewriter(stream);
    tokens.insertBefore(1, "x");
    tokens.insertBefore(2, "y");
    tokens.insertBefore(0, "z");
    String result = tokens.getText();

View Full Code Here

                       "A : 'a';\n" +
                       "B : 'b';\n" +
                       "C : 'c';\n");
    String input = "abcc";
    LexerInterpreter lexEngine = g.createLexerInterpreter(new ANTLRInputStream(input));
    CommonTokenStream stream = new CommonTokenStream(lexEngine);
    stream.fill();
    TokenStreamRewriter tokens = new TokenStreamRewriter(stream);
    tokens.replace(1, 2, "foo");
    tokens.replace(0, 3, "bar");
    stream.fill();
// wipes prior nested replace
    String result = tokens.getText();
    String expecting = "bar";
    assertEquals(expecting, result);
  }

View Full Code Here

                       "A : 'a';\n" +
                       "B : 'b';\n" +
                       "C : 'c';\n");
    String input = "abcc";
    LexerInterpreter lexEngine = g.createLexerInterpreter(new ANTLRInputStream(input));
    CommonTokenStream stream = new CommonTokenStream(lexEngine);
    stream.fill();
    TokenStreamRewriter tokens = new TokenStreamRewriter(stream);
    tokens.replace(0, 3, "bar");
    tokens.replace(1, 2, "foo");
    stream.fill();
// cannot split earlier replace
    Exception exc = null;
    try {
      tokens.getText();
    }

View Full Code Here

                       "A : 'a';\n" +
                       "B : 'b';\n" +
                       "C : 'c';\n");
    String input = "abcc";
    LexerInterpreter lexEngine = g.createLexerInterpreter(new ANTLRInputStream(input));
    CommonTokenStream stream = new CommonTokenStream(lexEngine);
    stream.fill();
    TokenStreamRewriter tokens = new TokenStreamRewriter(stream);
    tokens.replace(1, 2, "foo");
    tokens.replace(0, 2, "bar");
    stream.fill();
// wipes prior nested replace
    String result = tokens.getText();
    String expecting = "barc";
    assertEquals(expecting, result);
  }

View Full Code Here

                       "A : 'a';\n" +
                       "B : 'b';\n" +
                       "C : 'c';\n");
    String input = "abcc";
    LexerInterpreter lexEngine = g.createLexerInterpreter(new ANTLRInputStream(input));
    CommonTokenStream stream = new CommonTokenStream(lexEngine);
    stream.fill();
    TokenStreamRewriter tokens = new TokenStreamRewriter(stream);
    tokens.replace(1, 2, "foo");
    tokens.replace(1, 3, "bar");
    stream.fill();
// wipes prior nested replace
    String result = tokens.getText();
    String expecting = "abar";
    assertEquals(expecting, result);
  }

View Full Code Here

                       "A : 'a';\n" +
                       "B : 'b';\n" +
                       "C : 'c';\n");
    String input = "abcc";
    LexerInterpreter lexEngine = g.createLexerInterpreter(new ANTLRInputStream(input));
    CommonTokenStream stream = new CommonTokenStream(lexEngine);
    stream.fill();
    TokenStreamRewriter tokens = new TokenStreamRewriter(stream);
    tokens.replace(1, 2, "foo");
    tokens.replace(1, 2, "foo");
    stream.fill();
// drop previous, identical
    String result = tokens.getText();
    String expecting = "afooc";
    assertEquals(expecting, result);
  }

View Full Code Here

                       "A : 'a';\n" +
                       "B : 'b';\n" +
                       "C : 'c';\n");
    String input = "abc";
    LexerInterpreter lexEngine = g.createLexerInterpreter(new ANTLRInputStream(input));
    CommonTokenStream stream = new CommonTokenStream(lexEngine);
    stream.fill();
    TokenStreamRewriter tokens = new TokenStreamRewriter(stream);
    tokens.insertBefore(1, "foo");
    tokens.replace(1, 2, "foo");
    stream.fill();
// kill prev insert
    String result = tokens.getText();
    String expecting = "afoofoo";
    assertEquals(expecting, result);
  }

View Full Code Here

                       "A : 'a';\n" +
                       "B : 'b';\n" +
                       "C : 'c';\n");
    String input = "abcc";
    LexerInterpreter lexEngine = g.createLexerInterpreter(new ANTLRInputStream(input));
    CommonTokenStream stream = new CommonTokenStream(lexEngine);
    stream.fill();
    TokenStreamRewriter tokens = new TokenStreamRewriter(stream);
    tokens.insertBefore(1, "x");
    tokens.replace(2, 3, "foo");
    String result = tokens.getText();
    String expecting = "axbfoo";

View Full Code Here

                       "A : 'a';\n" +
                       "B : 'b';\n" +
                       "C : 'c';\n");
    String input = "abcc";
    LexerInterpreter lexEngine = g.createLexerInterpreter(new ANTLRInputStream(input));
    CommonTokenStream stream = new CommonTokenStream(lexEngine);
    stream.fill();
    TokenStreamRewriter tokens = new TokenStreamRewriter(stream);
    tokens.replace(2, 3, "foo");
    tokens.insertBefore(1, "x");
    String result = tokens.getText();
    String expecting = "axbfoo";

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.antlr.v4.runtime.CommonTokenStream

com.espertech.esper.core.deploy.EPLModuleUtil

com.espertech.esper.event.property.PropertyParser

org.antlr.runtime.ANTLRStringStream

org.antlr.runtime.CharStream

org.antlr.runtime.CommonToken

org.antlr.runtime.Token

org.antlr.runtime.tree.CommonTree

org.antlr.runtime.tree.CommonTreeNodeStream

org.antlr.runtime.tree.Tree

org.antlr.runtime.tree.TreeVisitor

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.