Examples of TokenSource

com.dotcms.repackage.org.antlr.runtime.TokenSource
org.antlr.runtime.TokenSource
A source of tokens must provide a sequence of tokens via nextToken() and also must reveal it's source of characters; CommonToken's text is computed from a CharStream; it only store indices into the char stream. Errors from the lexer are never passed to the parser. Either you want to keep going or you do not upon token recognition error. If you do not want to continue lexing then you do not want to continue parsing. Just throw an exception not under RecognitionException and Java will naturally toss you all the way out of the recognizers. If you want to continue lexing then you should not throw an exception to the parser--it has already requested a token. Keep lexing until you get a valid one. Just report errors and keep going, looking for a valid token.
org.antlr.v4.runtime.TokenSource
A source of tokens must provide a sequence of tokens via {@link #nextToken()}and also must reveal it's source of characters; {@link CommonToken}'s text is computed from a {@link CharStream}; it only store indices into the char stream.
Errors from the lexer are never passed to the parser. Either you want to keep going or you do not upon token recognition error. If you do not want to continue lexing then you do not want to continue parsing. Just throw an exception not under {@link RecognitionException} and Java will naturally tossyou all the way out of the recognizers. If you want to continue lexing then you should not throw an exception to the parser--it has already requested a token. Keep lexing until you get a valid one. Just report errors and keep going, looking for a valid token.

Examples of org.antlr.runtime.TokenSource

        return partialStatement;
    }


    public static String squeezeStatement(String sql)
    {
        TokenSource tokens = getLexer(sql, ImmutableSet.<String>of());
        StringBuilder sb = new StringBuilder();
        while (true) {
            Token token = tokens.nextToken();
            if (token.getType() == Token.EOF) {
                break;
            }
            if (token.getType() == StatementLexer.WS) {
                sb.append(' ');

View Full Code Here

Examples of org.antlr.runtime.TokenSource

        return sb.toString().trim();
    }


    public static boolean isEmptyStatement(String sql)
    {
        TokenSource tokens = getLexer(sql, ImmutableSet.<String>of());
        while (true) {
            Token token = tokens.nextToken();
            if (token.getType() == Token.EOF) {
                return true;
            }
            if (token.getChannel() != Token.HIDDEN_CHANNEL) {
                return false;

View Full Code Here

Examples of org.antlr.runtime.TokenSource

        this(sql, ImmutableSet.of(";"));
    }


    public StatementSplitter(String sql, Set<String> delimiters)
    {
        TokenSource tokens = getLexer(sql, delimiters);
        ImmutableList.Builder<Statement> list = ImmutableList.builder();
        StringBuilder sb = new StringBuilder();
        while (true) {
            Token token = tokens.nextToken();
            if (token.getType() == Token.EOF) {
                break;
            }
            if (token.getType() == StatementLexer.TERMINATOR) {
                String statement = sb.toString().trim();

View Full Code Here

Examples of org.antlr.runtime.TokenSource

        return partialStatement;
    }


    public static String squeezeStatement(String sql)
    {
        TokenSource tokens = getLexer(sql, ImmutableSet.<String>of());
        StringBuilder sb = new StringBuilder();
        int index = 0;
        while (true) {
            Token token;
            try {
                token = tokens.nextToken();
                index = ((CommonToken) token).getStopIndex() + 1;
            }
            catch (ParsingException e) {
                sb.append(sql.substring(index));
                break;

View Full Code Here

Examples of org.antlr.runtime.TokenSource

    }


    @Override
    public CommonTree parseStatement() throws RecognitionException {
        CharStream input = new ANTLRStringStream(statement);
        TokenSource lexer = new CmisQlStrictLexer(input);
        tokens = new CommonTokenStream(lexer);
        CmisQlStrictParser parser = new CmisQlStrictParser(tokens);


        query_return parsedStatement = parser.query();
        if (parser.hasErrors()) {

View Full Code Here

Examples of org.antlr.runtime.TokenSource

     *      
     * @throws RecognitionException
     */
    public static CmisQueryWalker getWalker(String statement) throws RecognitionException {
        CharStream input = new ANTLRStringStream(statement);
        TokenSource lexer = new CmisQlStrictLexer(input);
        TokenStream tokens = new CommonTokenStream(lexer);
        CmisQlStrictParser parser = new CmisQlStrictParser(tokens);
        CommonTree parserTree; // the ANTLR tree after parsing phase


        query_return parsedStatement = parser.query();

View Full Code Here

Examples of org.antlr.v4.runtime.TokenSource

  protected TokenStream createTokenStream(TokenSource src) {
    return new CommonTokenStream(src);
  }


  @Test public void testOffChannel() throws Exception {
        TokenSource lexer = // simulate input " x =34  ;\n"
            new TokenSource() {
                int i = 0;
                WritableToken[] tokens = {
                    new CommonToken(1," ") {{channel = Lexer.HIDDEN;}},
                    new CommonToken(1,"x"),
                    new CommonToken(1," ") {{channel = Lexer.HIDDEN;}},

View Full Code Here

Examples of org.antlr.v4.runtime.TokenSource

        assertEquals("=", tokens.LT(-3).getText());
        assertEquals("x", tokens.LT(-4).getText());
    }


  @Test public void testFetchOffChannel() throws Exception {
    TokenSource lexer = // simulate input " x =34  ; \n"
                        // token indexes   01234 56789
      new TokenSource() {
        int i = 0;
        WritableToken[] tokens = {
        new CommonToken(1," ") {{channel = Lexer.HIDDEN;}}, // 0
        new CommonToken(1,"x"),                // 1
        new CommonToken(1," ") {{channel = Lexer.HIDDEN;}},  // 2

View Full Code Here

Examples of org.antlr.v4.runtime.TokenSource

    assertEquals(null, tokens.getHiddenTokensToRight(9));
  }


  @Test
  public void testSingleEOF() throws Exception {
    TokenSource lexer = new TokenSource() {


      @Override
      public Token nextToken() {
        return new CommonToken(Token.EOF);
      }

View Full Code Here

Examples of org.antlr.v4.runtime.TokenSource

    assertEquals(1, tokens.size());
  }


  @Test(expected = IllegalStateException.class)
  public void testCannotConsumeEOF() throws Exception {
    TokenSource lexer = new TokenSource() {


      @Override
      public Token nextToken() {
        return new CommonToken(Token.EOF);
      }

View Full Code Here

0 1 2 3

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.