Examples of org.antlr.v4.runtime.CommonTokenStream

org.antlr.v4.runtime.tree.ParseTreeWalker
This class extends {@link BufferedTokenStream} with functionality to filtertoken streams to tokens on a particular channel (tokens where {@link Token#getChannel} returns a particular value).
This token stream provides access to all tokens by index or when calling methods like {@link #getText}. The channel filtering is only used for code accessing tokens via the lookahead methods {@link #LA}, {@link #LT}, and {@link #LB}.

By default, tokens are placed on the default channel ( {@link Token#DEFAULT_CHANNEL}), but may be reassigned by using the {@code ->channel(HIDDEN)} lexer command, or by using an embedded action tocall {@link Lexer#setChannel}.

Note: lexer rules which use the {@code ->skip} lexer command or call{@link Lexer#skip} do not produce tokens at all, so input text matched bysuch a rule will not be available as part of the token stream, regardless of channel.

    try {
      if ( !quiet ) System.err.println(f);
      // Create a scanner that reads from the input stream passed to us
      Lexer lexer = new Java8Lexer(new ANTLRFileStream(f));


      CommonTokenStream tokens = new CommonTokenStream(lexer);
//      long start = System.currentTimeMillis();
//      tokens.fill(); // load all and check time
//      long stop = System.currentTimeMillis();
//      lexerTime += stop-start;

View Full Code Here

    // create new lexer and token stream for each line (expression)
    ANTLRInputStream input = new ANTLRInputStream(str + "\n"); //$NON-NLS-1$
    CommandsLexer lexer = new CommandsLexer(input);
    lexer.setLine(line); // notify lexer of input position
    lexer.setCharPositionInLine(0);
    CommonTokenStream tokens = new CommonTokenStream(lexer);
    parser.setInputStream(tokens); // notify parser of new token stream
    lexer.reset();
    return parser.command(); // start the parser
  }

View Full Code Here


        JetTemplateLexer lexer = new JetTemplateLexer(is);
        lexer.removeErrorListeners(); // remove ConsoleErrorListener
        lexer.addErrorListener(JetTemplateErrorListener.getInstance());


        JetTemplateParser parser = new JetTemplateParser(new CommonTokenStream(lexer));
        parser.removeErrorListeners(); // remove ConsoleErrorListener
        parser.addErrorListener(JetTemplateErrorListener.getInstance());
        parser.setErrorHandler(new JetTemplateErrorStrategy());


        TemplateContext templateParseTree = parser.template();

View Full Code Here

//    lexer.setTokenFactory(factory);
    CommonTokenStream tokens = new CommonTokenStream(lexer);
    
    MsonParser parser = new MsonParser(tokens);
//    parser.setTokenFactory(factory);
    parser.addErrorListener(new DiagnosticErrorListener());
    parser.getInterpreter().setPredictionMode(
        PredictionMode.LL_EXACT_AMBIG_DETECTION);
    parser.setBuildParseTree(true);
    ParseTree tree = parser.mson();
    // show tree in text form

View Full Code Here

    ClassUtils cu = new ClassUtils();
    setup(cu);
    registry.clearForType(JavaCompletionTypes.CUSTOM_TYPE);
    registry.clearForType(JavaCompletionTypes.FIELD);
    registry.clearForType(JavaCompletionTypes.NAME);
    Lexer lexer = new JavaLexer(new ANTLRInputStream(txt));
    CommonTokenStream tokens = new CommonTokenStream(lexer);


    // Create a parser that reads from the scanner
    JavaParser parser = new JavaParser(tokens);
    parser.removeErrorListeners();

View Full Code Here

    // Create a parser that reads from the scanner
    JavaParser parser = new JavaParser(tokens);
    parser.removeErrorListeners();
    
    // start parsing at the compilationUnit rule
    ParserRuleContext t = parser.compilationUnit();
    ParseTreeWalker walker = new ParseTreeWalker();
    List<AutocompleteCandidate> q = new ArrayList<AutocompleteCandidate>();
          
    ImportDeclarationCompletion extractor = new ImportDeclarationCompletion(txt,cur,registry,cps,cu);
    NameBuilder extractor2 = new NameBuilder(registry,cu );

View Full Code Here

          }
        }
      }
      if(arg0.getParent() instanceof ExpressionContext) {
        // we are the leftmost child of the expression
        ParseTree chld = arg0.getParent().getChild(arg0.getParent().getChildCount()-1);
        if(!chld.equals(arg0)) return;
        addQuery(classUtils.expandExpression(arg0.getParent().getText(), registry));
      }
    }
  }

View Full Code Here

//    parser.setTokenFactory(factory);
    parser.addErrorListener(new DiagnosticErrorListener());
    parser.getInterpreter().setPredictionMode(
        PredictionMode.LL_EXACT_AMBIG_DETECTION);
    parser.setBuildParseTree(true);
    ParseTree tree = parser.mson();
    // show tree in text form
    // System.out.println(tree.toStringTree(parser));


    ParseTreeWalker walker = new ParseTreeWalker();
    SymbolTable symtab = new SymbolTable();

View Full Code Here

    JavaParser parser = new JavaParser(tokens);
    parser.removeErrorListeners();
    
    // start parsing at the compilationUnit rule
    ParserRuleContext t = parser.compilationUnit();
    ParseTreeWalker walker = new ParseTreeWalker();
    List<AutocompleteCandidate> q = new ArrayList<AutocompleteCandidate>();
          
    ImportDeclarationCompletion extractor = new ImportDeclarationCompletion(txt,cur,registry,cps,cu);
    NameBuilder extractor2 = new NameBuilder(registry,cu );
    NodeCompletion extractor3 = new NodeCompletion(txt,cur, registry, cu);
    walker.walk(extractor, t);
    if(extractor.getQuery()!=null)
      q.addAll(extractor.getQuery());
    walker.walk(extractor2, t);
    walker.walk(extractor3, t);
    if(extractor3.getQuery()!=null)
      q.addAll(extractor3.getQuery());
    List<String> ret = registry.searchCandidates(q);


    // this shows the GUI

View Full Code Here

    parser.setBuildParseTree(true);
    ParseTree tree = parser.mson();
    // show tree in text form
    // System.out.println(tree.toStringTree(parser));


    ParseTreeWalker walker = new ParseTreeWalker();
    SymbolTable symtab = new SymbolTable();
    DefPhase def = new DefPhase(symtab);
    walker.walk(def, tree);
    // create next phase and feed symbol table info from def to ref phase
    RefPhase ref = new RefPhase(symtab, def.scopes);
    walker.walk(ref, tree);
  }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.antlr.v4.runtime.CommonTokenStream

com.espertech.esper.core.deploy.EPLModuleUtil

com.espertech.esper.event.property.PropertyParser

org.antlr.runtime.ANTLRStringStream

org.antlr.runtime.CharStream

org.antlr.runtime.CommonToken

org.antlr.runtime.Token

org.antlr.runtime.tree.CommonTree

org.antlr.runtime.tree.CommonTreeNodeStream

org.antlr.runtime.tree.Tree

org.antlr.runtime.tree.TreeVisitor

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.