Examples of org.antlr.v4.runtime.CommonTokenStream

org.antlr.v4.runtime.CommonTokenStream
This class extends {@link BufferedTokenStream} with functionality to filtertoken streams to tokens on a particular channel (tokens where {@link Token#getChannel} returns a particular value).
This token stream provides access to all tokens by index or when calling methods like {@link #getText}. The channel filtering is only used for code accessing tokens via the lookahead methods {@link #LA}, {@link #LT}, and {@link #LB}.

By default, tokens are placed on the default channel ( {@link Token#DEFAULT_CHANNEL}), but may be reassigned by using the {@code ->channel(HIDDEN)} lexer command, or by using an embedded action tocall {@link Lexer#setChannel}.

Note: lexer rules which use the {@code ->skip} lexer command or call{@link Lexer#skip} do not produce tokens at all, so input text matched bysuch a rule will not be available as part of the token stream, regardless of channel.

    parser.setProfile(true);
    for (String s : input) {
      lexEngine.reset();
      parser.reset();
      lexEngine.setInputStream(new ANTLRInputStream(s));
      CommonTokenStream tokens = new CommonTokenStream(lexEngine);
      parser.setInputStream(tokens);
      Rule r = g.rules.get(startRule);
      if ( r==null ) {
        return parser.getParseInfo().getDecisionInfo();
      }

View Full Code Here


  protected void process(Lexer lexer, Class<? extends Parser> parserClass, Parser parser, InputStream is, Reader r) throws IOException, IllegalAccessException, InvocationTargetException, PrintException {
    try {
      ANTLRInputStream input = new ANTLRInputStream(r);
      lexer.setInputStream(input);
      CommonTokenStream tokens = new CommonTokenStream(lexer);


      tokens.fill();


      if ( showTokens ) {
        for (Object tok : tokens.getTokens()) {
          System.out.println(tok);
        }
      }


      if ( startRuleName.equals(LEXER_START_RULE_NAME) ) return;

View Full Code Here

            final Constructor<? extends Lexer> lexerCtor = lexerClass.getConstructor(CharStream.class);
            final Constructor<? extends Parser> parserCtor = parserClass.getConstructor(TokenStream.class);


            // construct initial instances of the lexer and parser to deserialize their ATNs
            TokenSource tokenSource = lexerCtor.newInstance(new ANTLRInputStream(""));
            parserCtor.newInstance(new CommonTokenStream(tokenSource));


            return new ParserFactory() {
        @Override
                public FileParseResult parseFile(CharStream input, int currentPass, int thread) {
          final Checksum checksum = new CRC32();


          final long startTime = System.nanoTime();
          assert thread >= 0 && thread < NUMBER_OF_THREADS;


                    try {
            ParseTreeListener listener = sharedListeners[thread];
            if (listener == null) {
              listener = listenerClass.newInstance();
              sharedListeners[thread] = listener;
            }


            Lexer lexer = sharedLexers[thread];
                        if (REUSE_LEXER && lexer != null) {
                            lexer.setInputStream(input);
                        } else {
              Lexer previousLexer = lexer;
                            lexer = lexerCtor.newInstance(input);
              DFA[] decisionToDFA = (FILE_GRANULARITY || previousLexer == null ? lexer : previousLexer).getInterpreter().decisionToDFA;
              if (!REUSE_LEXER_DFA || (!FILE_GRANULARITY && previousLexer == null)) {
                decisionToDFA = new DFA[decisionToDFA.length];
              }


              if (COMPUTE_TRANSITION_STATS) {
                lexer.setInterpreter(new StatisticsLexerATNSimulator(lexer, lexer.getATN(), decisionToDFA, lexer.getInterpreter().getSharedContextCache()));
              } else if (!REUSE_LEXER_DFA) {
                lexer.setInterpreter(new LexerATNSimulator(lexer, lexer.getATN(), decisionToDFA, lexer.getInterpreter().getSharedContextCache()));
              }


              sharedLexers[thread] = lexer;
                        }


            lexer.removeErrorListeners();
            lexer.addErrorListener(DescriptiveErrorListener.INSTANCE);


            if (lexer.getInterpreter().decisionToDFA[0] == null) {
              ATN atn = lexer.getATN();
              for (int i = 0; i < lexer.getInterpreter().decisionToDFA.length; i++) {
                lexer.getInterpreter().decisionToDFA[i] = new DFA(atn.getDecisionState(i), i);
              }
            }


                        CommonTokenStream tokens = new CommonTokenStream(lexer);
                        tokens.fill();
                        tokenCount.addAndGet(currentPass, tokens.size());


            if (COMPUTE_CHECKSUM) {
              for (Token token : tokens.getTokens()) {
                updateChecksum(checksum, token);
              }
            }


                        if (!RUN_PARSER) {
                            return new FileParseResult(input.getSourceName(), (int)checksum.getValue(), null, tokens.size(), startTime, lexer, null);
                        }


            final long parseStartTime = System.nanoTime();
            Parser parser = sharedParsers[thread];
                        if (REUSE_PARSER && parser != null) {
                            parser.setInputStream(tokens);
                        } else {
              Parser previousParser = parser;


              if (USE_PARSER_INTERPRETER) {
                Parser referenceParser = parserCtor.newInstance(tokens);
                parser = new ParserInterpreter(referenceParser.getGrammarFileName(), referenceParser.getVocabulary(), Arrays.asList(referenceParser.getRuleNames()), referenceParser.getATN(), tokens);
              }
              else {
                parser = parserCtor.newInstance(tokens);
              }


              DFA[] decisionToDFA = (FILE_GRANULARITY || previousParser == null ? parser : previousParser).getInterpreter().decisionToDFA;
              if (!REUSE_PARSER_DFA || (!FILE_GRANULARITY && previousParser == null)) {
                decisionToDFA = new DFA[decisionToDFA.length];
              }


              if (COMPUTE_TRANSITION_STATS) {
                parser.setInterpreter(new StatisticsParserATNSimulator(parser, parser.getATN(), decisionToDFA, parser.getInterpreter().getSharedContextCache()));
              } else if (!REUSE_PARSER_DFA) {
                parser.setInterpreter(new ParserATNSimulator(parser, parser.getATN(), decisionToDFA, parser.getInterpreter().getSharedContextCache()));
              }


              sharedParsers[thread] = parser;
                        }


            parser.removeParseListeners();
            parser.removeErrorListeners();
            if (!TWO_STAGE_PARSING) {
              parser.addErrorListener(DescriptiveErrorListener.INSTANCE);
              parser.addErrorListener(new SummarizingDiagnosticErrorListener());
            }


            if (parser.getInterpreter().decisionToDFA[0] == null) {
              ATN atn = parser.getATN();
              for (int i = 0; i < parser.getInterpreter().decisionToDFA.length; i++) {
                parser.getInterpreter().decisionToDFA[i] = new DFA(atn.getDecisionState(i), i);
              }
            }


            parser.getInterpreter().setPredictionMode(TWO_STAGE_PARSING ? PredictionMode.SLL : PREDICTION_MODE);
            parser.setBuildParseTree(BUILD_PARSE_TREES);
            if (!BUILD_PARSE_TREES && BLANK_LISTENER) {
              parser.addParseListener(listener);
            }
            if (BAIL_ON_ERROR || TWO_STAGE_PARSING) {
              parser.setErrorHandler(new BailErrorStrategy());
            }


                        Method parseMethod = parserClass.getMethod(entryPoint);
                        Object parseResult;


            try {
              if (COMPUTE_CHECKSUM && !BUILD_PARSE_TREES) {
                parser.addParseListener(new ChecksumParseTreeListener(checksum));
              }


              if (USE_PARSER_INTERPRETER) {
                ParserInterpreter parserInterpreter = (ParserInterpreter)parser;
                parseResult = parserInterpreter.parse(Collections.lastIndexOfSubList(Arrays.asList(parser.getRuleNames()), Collections.singletonList(entryPoint)));
              }
              else {
                parseResult = parseMethod.invoke(parser);
              }
            } catch (InvocationTargetException ex) {
              if (!TWO_STAGE_PARSING) {
                throw ex;
              }


              String sourceName = tokens.getSourceName();
              sourceName = sourceName != null && !sourceName.isEmpty() ? sourceName+": " : "";
              if (REPORT_SECOND_STAGE_RETRY) {
                System.err.println(sourceName+"Forced to retry with full context.");
              }


              if (!(ex.getCause() instanceof ParseCancellationException)) {
                throw ex;
              }


              tokens.reset();
              if (REUSE_PARSER && parser != null) {
                parser.setInputStream(tokens);
              } else {
                Parser previousParser = parser;


                if (USE_PARSER_INTERPRETER) {
                  Parser referenceParser = parserCtor.newInstance(tokens);
                  parser = new ParserInterpreter(referenceParser.getGrammarFileName(), referenceParser.getVocabulary(), Arrays.asList(referenceParser.getRuleNames()), referenceParser.getATN(), tokens);
                }
                else {
                  parser = parserCtor.newInstance(tokens);
                }


                DFA[] decisionToDFA = previousParser.getInterpreter().decisionToDFA;
                if (COMPUTE_TRANSITION_STATS) {
                  parser.setInterpreter(new StatisticsParserATNSimulator(parser, parser.getATN(), decisionToDFA, parser.getInterpreter().getSharedContextCache()));
                } else if (!REUSE_PARSER_DFA) {
                  parser.setInterpreter(new ParserATNSimulator(parser, parser.getATN(), decisionToDFA, parser.getInterpreter().getSharedContextCache()));
                }


                sharedParsers[thread] = parser;
              }


              parser.removeParseListeners();
              parser.removeErrorListeners();
              parser.addErrorListener(DescriptiveErrorListener.INSTANCE);
              parser.addErrorListener(new SummarizingDiagnosticErrorListener());
              parser.getInterpreter().setPredictionMode(PredictionMode.LL);
              parser.setBuildParseTree(BUILD_PARSE_TREES);
              if (COMPUTE_CHECKSUM && !BUILD_PARSE_TREES) {
                parser.addParseListener(new ChecksumParseTreeListener(checksum));
              }
              if (!BUILD_PARSE_TREES && BLANK_LISTENER) {
                parser.addParseListener(listener);
              }
              if (BAIL_ON_ERROR) {
                parser.setErrorHandler(new BailErrorStrategy());
              }


              parseResult = parseMethod.invoke(parser);
            }


            assertThat(parseResult, instanceOf(ParseTree.class));
            if (COMPUTE_CHECKSUM && BUILD_PARSE_TREES) {
              ParseTreeWalker.DEFAULT.walk(new ChecksumParseTreeListener(checksum), (ParseTree)parseResult);
            }
                        if (BUILD_PARSE_TREES && BLANK_LISTENER) {
                            ParseTreeWalker.DEFAULT.walk(listener, (ParseTree)parseResult);
                        }


            return new FileParseResult(input.getSourceName(), (int)checksum.getValue(), (ParseTree)parseResult, tokens.size(), TIME_PARSE_ONLY ? parseStartTime : startTime, lexer, parser);
                    } catch (Exception e) {
            if (!REPORT_SYNTAX_ERRORS && e instanceof ParseCancellationException) {
              return new FileParseResult("unknown", (int)checksum.getValue(), null, 0, startTime, null, null);
            }

View Full Code Here

    Constructor<? extends Lexer> ctor = c.getConstructor(CharStream.class);
    Lexer lexer = ctor.newInstance(in);


    Class<? extends Parser> pc = parserClass.asSubclass(Parser.class);
    Constructor<? extends Parser> pctor = pc.getConstructor(TokenStream.class);
    CommonTokenStream tokens = new CommonTokenStream(lexer);
    Parser parser = pctor.newInstance(tokens);
    return new Pair<Parser, Lexer>(parser, lexer);
  }

View Full Code Here

                       "A : 'a';\n" +
                       "B : 'b';\n" +
                       "C : 'c';\n");
    String input = "abc";
    LexerInterpreter lexEngine = g.createLexerInterpreter(new ANTLRInputStream(input));
    CommonTokenStream stream = new CommonTokenStream(lexEngine);
    stream.fill();
    TokenStreamRewriter tokens = new TokenStreamRewriter(stream);
    tokens.replace(0, "x");
    String result = tokens.getText();
    String expecting = "xbc";
    assertEquals(expecting, result);

View Full Code Here

                       "A : 'a';\n" +
                       "B : 'b';\n" +
                       "C : 'c';\n");
    String input = "abc";
    LexerInterpreter lexEngine = g.createLexerInterpreter(new ANTLRInputStream(input));
    CommonTokenStream stream = new CommonTokenStream(lexEngine);
    stream.fill();
    TokenStreamRewriter tokens = new TokenStreamRewriter(stream);
    tokens.replace(2, "x");
    String result = tokens.getText();
    String expecting = "abx";
    assertEquals(expecting, result);

View Full Code Here

                       "A : 'a';\n" +
                       "B : 'b';\n" +
                       "C : 'c';\n");
    String input = "abc";
    LexerInterpreter lexEngine = g.createLexerInterpreter(new ANTLRInputStream(input));
    CommonTokenStream stream = new CommonTokenStream(lexEngine);
    stream.fill();
    TokenStreamRewriter tokens = new TokenStreamRewriter(stream);
    tokens.replace(1, "x");
    String result = tokens.getText();
    String expecting = "axc";
    assertEquals(expecting, result);

View Full Code Here

                       "WS : ' '+;\n");
    // Tokens: 0123456789
    // Input:  x = 3 * 0;
    String input = "x = 3 * 0;";
    LexerInterpreter lexEngine = g.createLexerInterpreter(new ANTLRInputStream(input));
    CommonTokenStream stream = new CommonTokenStream(lexEngine);
    stream.fill();
    TokenStreamRewriter tokens = new TokenStreamRewriter(stream);
    tokens.replace(4, 8, "0");
    stream.fill();
// replace 3 * 0 with 0


    String result = tokens.getTokenStream().getText();
    String expecting = "x = 3 * 0;";
    assertEquals(expecting, result);

View Full Code Here

                       "WS : ' '+;\n");
    // Tokens: 012345678901234567
    // Input:  x = 3 * 0 + 2 * 0;
    String input = "x = 3 * 0 + 2 * 0;";
    LexerInterpreter lexEngine = g.createLexerInterpreter(new ANTLRInputStream(input));
    CommonTokenStream stream = new CommonTokenStream(lexEngine);
    stream.fill();
    TokenStreamRewriter tokens = new TokenStreamRewriter(stream);


    String result = tokens.getTokenStream().getText();
    String expecting = "x = 3 * 0 + 2 * 0;";
    assertEquals(expecting, result);


    tokens.replace(4, 8, "0");
    stream.fill();
// replace 3 * 0 with 0
    result = tokens.getText();
    expecting = "x = 0 + 2 * 0;";
    assertEquals(expecting, result);


    result = tokens.getText(Interval.of(0, 17));
    expecting = "x = 0 + 2 * 0;";
    assertEquals(expecting, result);


    result = tokens.getText(Interval.of(4, 8));
    expecting = "0";
    assertEquals(expecting, result);


    result = tokens.getText(Interval.of(0, 8));
    expecting = "x = 0";
    assertEquals(expecting, result);


    result = tokens.getText(Interval.of(12, 16));
    expecting = "2 * 0";
    assertEquals(expecting, result);


    tokens.insertAfter(17, "// comment");
    result = tokens.getText(Interval.of(12, 18));
    expecting = "2 * 0;// comment";
    assertEquals(expecting, result);


    result = tokens.getText(Interval.of(0, 8));
    stream.fill();
// try again after insert at end
    expecting = "x = 0";
    assertEquals(expecting, result);
  }

View Full Code Here

                       "A : 'a';\n" +
                       "B : 'b';\n" +
                       "C : 'c';\n");
    String input = "abc";
    LexerInterpreter lexEngine = g.createLexerInterpreter(new ANTLRInputStream(input));
    CommonTokenStream stream = new CommonTokenStream(lexEngine);
    stream.fill();
    TokenStreamRewriter tokens = new TokenStreamRewriter(stream);
    tokens.replace(1, "x");
    tokens.replace(1, "y");
    String result = tokens.getText();
    String expecting = "ayc";

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.antlr.v4.runtime.CommonTokenStream

com.espertech.esper.core.deploy.EPLModuleUtil

com.espertech.esper.event.property.PropertyParser

org.antlr.runtime.ANTLRStringStream

org.antlr.runtime.CharStream

org.antlr.runtime.CommonToken

org.antlr.runtime.Token

org.antlr.runtime.tree.CommonTree

org.antlr.runtime.tree.CommonTreeNodeStream

org.antlr.runtime.tree.Tree

org.antlr.runtime.tree.TreeVisitor

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.