Examples of de.fosd.typechef.lexer.Token

de.fosd.typechef.lexer.Token
A Preprocessor token. @see Preprocessor

        }
        return tok;
    }


    private Token parse_undef() throws IOException, LexerException {
        Token tok = source_token_nonwhite();
        if (tok.getType() != IDENTIFIER) {
            error(tok, "Expected identifier, not " + tok.getText());
            if (tok.getType() == NL || tok.getType() == EOF)
                return tok;
        } else {
            // Macro m = macros.get(tok.getText());
            // if (m != null) {
            /* XXX error if predefined */
            macros = macros.undefine(tok.getText(), state
                    .getFullPresenceCondition());
            // }
        }
        return source_skipline(true);
    }

View Full Code Here

            LexerException {
        LexerSource lexer = (LexerSource) sourceManager.getSource();
        try {
            lexer.setInclude(true);
            processing_include = true;
            Token tok = getNextNonwhiteToken();


            String name;
            boolean quoted;
            StringBuilder buf = new StringBuilder();


            if (tok.getType() == STRING) {
                /*
                     * XXX Use the original text, not the value. Backslashes must
                     * not be treated as escapes here.
                     * PG: the above is no more needed, because the lexer does
                     * not handle backslashes as escapes when processing includes.
                     */
                buf.append((String) tok.getValue());
                HEADER:
                for (; ; ) {
                    tok = getNextToken();
                    switch (tok.getType()) {
                        case STRING:
                            //XXX: PG: That's forbidden by the standard (see 6.10.2 and 5.1.1.2)
                            buf.append((String) tok.getValue());
                            break;
                        case WHITESPACE:// ignore whitespace and comments for now
                            // ChK
                        case CPPCOMMENT:
                        case CCOMMENT:
                            break;
                        case NL:
                        case EOF:
                            break HEADER;
                        default:
                            warning(tok, "Unexpected token on #" + "include line");
                            return source_skipline(false);
                    }
                }
                quoted = true;
            } else if (tok.getType() == HEADER) {
                buf.append((String) tok.getValue());
                quoted = false;
                tok = source_skipline(true);
            } else if (tok.getType() == '<') {
                quoted = false;
                HEADER:
                for (; ; ) {
                    tok = getNextToken();
                    switch (tok.getType()) {
                        case '>':
                            break HEADER;
                        // XXX: PG: don't ignore WHITESPACE or CCOMMENT for now, I don't think they can be there;
                        // recheck!
                        case WHITESPACE:
                        case CCOMMENT:
                            warning(tok, "Unexpected token \"" + tok.getText() + "\" on #" + "include line");
                            return source_skipline(false);


                        case '.':
                        case '/':
                            buf.append((char) tok.getType());
                            break;
                        case INTEGER:
                        case IDENTIFIER:
                        default:
                            buf.append(getTextOrDefault(tok, "<Feature Expression>"));
                            break;
                    }
                }
            } else {
                error(tok, "Expected string or header, not " + getTextOrDefault(tok, "<Feature Expression>"));
                switch (tok.getType()) {
                    case NL:
                    case EOF:
                        return tok;
                    default:
                        /* Only if not a NL or EOF already. */
                        return source_skipline(false);
                }
            }


            name = buf.toString();
            processing_include = false;
            /* Do the inclusion. */
            include(sourceManager.getSource().getPath(), tok.getLine(), name,
                    quoted, next);


            /*
                * 'tok' is the 'nl' after the include. We use it after the #line
                * directive.

View Full Code Here

        if (!"pack".equals(pragmaName))
            warning(nameTok, "Unknown #" + "pragma: " + pragmaName);
    }


    private Token parse_pragma() throws IOException, LexerException {
        Token name;


        NAME:
        for (; ; ) {
            Token tok = getNextToken();
            switch (tok.getType()) {
                case EOF:
                    /*
                      * There ought to be a newline before EOF. At least, in any
                      * skipline context.
                      */
                    /* XXX Are we sure about this? */
                    warning(tok, "End of file in #" + "pragma");
                    return tok;
                case NL:
                    /* This may contain one or more newlines. */
                    warning(tok, "Empty #" + "pragma");
                    return tok;
                case CCOMMENT:
                case CPPCOMMENT:
                case WHITESPACE:
                    continue NAME;
                case IDENTIFIER:
                    name = tok;
                    break NAME;
                default:
                    return source_skipline(false);
            }
        }


        Token tok;
        List<Token> value = new ArrayList<Token>();
        VALUE:
        for (; ; ) {
            tok = getNextToken();
            switch (tok.getType()) {
                case EOF:
                    /*
                      * There ought to be a newline before EOF. At least, in any
                      * skipline context.
                      */

View Full Code Here

    private Token parseErrorToken(Token pptok, boolean is_error)
            throws IOException, LexerException {
        StringBuilder buf = new StringBuilder();
        buf.append('#').append(pptok.getText()).append(' ');
        /* Peculiar construction to ditch first whitespace. */
        Token tok = source_token_nonwhite();
        ERROR:
        for (; ; ) {
            switch (tok.getType()) {
                case NL:
                case EOF:
                    buf.append('\n');
                    break ERROR;
                default:
                    buf.append(tok.getText());
                    break;
            }
            tok = retrieveTokenFromSource();
        }
        if (is_error)

View Full Code Here

      * (actually, I fear something more complicated is needed).
      */
    private Token expanded_token(boolean inlineCppExpression,
                                 boolean hack_definedActivated) throws IOException, LexerException {
        for (; ; ) {
            Token tok = retrieveTokenFromSource();
            // System.out.println("Source token is " + tok);
            if ("defined".equals(getTextOrDefault(tok, ""))
                    || (hack_definedCounter == 0 && "(".equals(getTextOrDefault(tok, ""))))
                hack_definedCounter = 0;
            else
                hack_definedCounter++;
            if (tok.getType() == IDENTIFIER
                    && (!hack_definedActivated || hack_definedCounter != 1)) {
                MacroExpansion<MacroData>[] m = macros.getApplicableMacroExpansions(tok
                        .getText(), state.getFullPresenceCondition());
                if (m.length > 0
                        && tok.mayExpand()
                        && sourceManager.getSource().mayExpand(tok.getText())
                        && macro_expandToken(tok.getText(), m, tok,
                        inlineCppExpression))
                    continue;
                return tok;
            }
            return tok;

View Full Code Here

        }
    }


    private Token expanded_token_nonwhite(boolean inlineCppExpression)
            throws IOException, LexerException {
        Token tok;
        do {
            tok = expanded_token(inlineCppExpression, false);
            // System.out.println("expanded token is " + tok);
        } while (tok.isWhite());
        return tok;
    }

View Full Code Here

    private boolean processing_include;


    //XXX:Rename as expr_token_get, and move in a small subclass.
    private Token expr_token(boolean inlineCppExpression) throws IOException,
            LexerException {
        Token tok = expr_token;


        if (tok != null) {
            // System.out.println("ungetting");
            expr_token = null;
        } else {

View Full Code Here

        /*
           * System.out.flush(); (new Exception("expr(" + priority +
           * ") called")).printStackTrace(); System.err.flush();
           */


        Token tok = expr_token(true);
        ExprOrValue lhs;


        // System.out.println("Expr lhs token is " + tok);


        switch (tok.getType()) {
            case Token.P_FEATUREEXPR:
                // only found internally: generated FeatureExprToken
                assert tok instanceof FeatureExprToken;
                assert expectExpr;
                lhs = new ExprOrValue(((FeatureExprToken) tok).getExpr());
                break;
            case '(':
                lhs = parse_featureExprOrValue(0, expectExpr);
                tok = expr_token(true);
                if (tok.getType() != ')') {
                    expr_untoken(tok);
                    error(tok, "missing ) in expression after " + lhs + ", found "
                            + tok + " instead");
                    return new ExprOrValue(FeatureExprLib.False(), FeatureExprLib.zero());
                }
                break;


            case '~':
                lhs = new ExprOrValue(FeatureExprLib.l().createComplement(parse_featureExprOrValue(11, false).assumeValue(tok)));
                break;
            case '!':
                lhs = new ExprOrValue(parse_featureExprOrValue(11, true).assumeExpression(tok).not());
                break;
            case '-':
                lhs = new ExprOrValue(FeatureExprLib.l().createNeg(parse_featureExprOrValue(11, false).assumeValue(tok)));
                break;
            case '+':
                lhs = new ExprOrValue((parse_featureExprOrValue(11, false).assumeValue(tok)));
                break;
            case INTEGER:
                lhs = new ExprOrValue(FeatureExprLib.l().createInteger(
                        ((Number) tok.getValue()).longValue()));
                break;
            case CHARACTER:
                lhs = new ExprOrValue(FeatureExprLib.l()
                        .createCharacter((Character) tok.getValue()));
                break;
            case IDENTIFIER:
                if (tok.getText().equals("___BASE___"))                //XXX: False code?
                    lhs = new ExprOrValue(FeatureExprLib.True());
                else if (tok.getText().equals("___DEAD___"))            //XXX: False code?
                    lhs = new ExprOrValue(FeatureExprLib.False());
                else if (tok.getText().equals("__IF__")) {
                    lhs = new ExprOrValue(parse_ifExpr(tok));
                } else if (tok.getText().equals("defined")) {
                    lhs = new ExprOrValue(parse_definedExpr(false));
                } else if (tok.getText().equals("definedEx")) {
                    lhs = new ExprOrValue(parse_definedExpr(true));
                } else {


                    if (isPotentialFlag(tok.getText())
                            && warnings.contains(Warning.UNDEF))
                        warning(tok, "Undefined token '" + tok.getText()
                                + "' encountered in conditional.");
                    lhs = new ExprOrValue(FeatureExprLib.False(), FeatureExprLib.zero());
                }
                break;


            default:
                expr_untoken(tok);
                error(tok, "Bad token in expression: " + getTextOrDefault(tok, "<Feature Expression>"));
                return new ExprOrValue(FeatureExprLib.False(), FeatureExprLib.zero());
        }


        EXPR:
        for (; ; ) {
            // System.out.println("expr: lhs is " + lhs + ", pri = " +
            // priority);
            Token op = expr_token(true);
            int pri = expr_priority(op); /* 0 if not a binop. */
            if (pri == 0 || priority >= pri) {
                expr_untoken(op);
                break EXPR;
            }
            // System.out.println("rhs token is " + rhs);
            switch (op.getType()) {
                case '/':
                    lhs = new ExprOrValue(FeatureExprLib.l().createDivision(lhs.assumeValue(tok), parse_featureExprOrValue(pri, false).assumeValue(tok)));
                    // if (rhs == 0) {
                    // error(op, "Division by zero");
                    // lhs = 0;
                    // } else {
                    // lhs = lhs / rhs;
                    // }
                    break;
                case '%':
                    lhs = new ExprOrValue(FeatureExprLib.l().createModulo(lhs.assumeValue(tok), parse_featureExprOrValue(pri, false).assumeValue(tok)));
                    // if (rhs == 0) {
                    // error(op, "Modulus by zero");
                    // lhs = 0;
                    // } else {
                    // lhs = lhs % rhs;
                    // }
                    break;
                case '*':
                    lhs = new ExprOrValue(FeatureExprLib.l().createMult(lhs.assumeValue(tok), parse_featureExprOrValue(pri, false).assumeValue(tok)));
                    break;
                case '+':
                    lhs = new ExprOrValue(FeatureExprLib.l().createPlus(lhs.assumeValue(tok), parse_featureExprOrValue(pri, false).assumeValue(tok)));
                    break;
                case '-':
                    lhs = new ExprOrValue(FeatureExprLib.l().createMinus(lhs.assumeValue(tok), parse_featureExprOrValue(pri, false).assumeValue(tok)));
                    break;
                case '<':
                    lhs = new ExprOrValue(FeatureExprLib.l().createLessThan(lhs.assumeValue(tok), parse_featureExprOrValue(pri, false).assumeValue(tok)));
                    // lhs < rhs ? 1 : 0;
                    break;
                case '>':
                    lhs = new ExprOrValue(FeatureExprLib.l().createGreaterThan(lhs.assumeValue(tok), parse_featureExprOrValue(pri, false).assumeValue(tok)));// lhs >
                    // rhs ?
                    // 1 : 0;
                    break;
                case '&':
                    lhs = new ExprOrValue(FeatureExprLib.l().createBitAnd(lhs.assumeValue(tok), parse_featureExprOrValue(pri, false).assumeValue(tok)));// lhs & rhs;
                    break;
                case '^':
                    lhs = new ExprOrValue(FeatureExprLib.l().createPwr(lhs.assumeValue(tok), parse_featureExprOrValue(pri, false).assumeValue(tok)));// lhs ^ rhs;
                    break;
                case '|':
                    lhs = new ExprOrValue(FeatureExprLib.l().createBitOr(lhs.assumeValue(tok), parse_featureExprOrValue(pri, false).assumeValue(tok)));// lhs | rhs;
                    break;


                case LSH:
                    lhs = new ExprOrValue(FeatureExprLib.l().createShiftLeft(lhs.assumeValue(tok), parse_featureExprOrValue(pri, false).assumeValue(tok)));// lhs <<
                    // rhs;
                    break;
                case RSH:
                    lhs = new ExprOrValue(FeatureExprLib.l().createShiftRight(lhs.assumeValue(tok), parse_featureExprOrValue(pri, false).assumeValue(tok)));// lhs >>
                    // rhs;
                    break;
                case LE:
                    lhs = new ExprOrValue(FeatureExprLib.l().createLessThanEquals(lhs.assumeValue(tok), parse_featureExprOrValue(pri, false).assumeValue(tok)));// lhs
                    // <=
                    // rhs ?
                    // 1 :
                    // 0;
                    break;
                case GE:
                    lhs = new ExprOrValue(FeatureExprLib.l().createGreaterThanEquals(lhs.assumeValue(tok), parse_featureExprOrValue(pri, false).assumeValue(tok)));// lhs
                    // >=
                    // rhs ?
                    // 1 :
                    // 0;
                    break;
                case EQ:
                    lhs = new ExprOrValue(FeatureExprLib.l().createEquals(lhs.assumeValue(tok), parse_featureExprOrValue(pri, false).assumeValue(tok)));// lhs == rhs
                    // ?
                    // 1 :
                    // 0;
                    break;
                case NE:
                    lhs = new ExprOrValue(FeatureExprLib.l().createNotEquals(lhs.assumeValue(tok), parse_featureExprOrValue(pri, false).assumeValue(tok)));// lhs !=
                    // rhs
                    // ?
                    // 1 : 0;
                    break;
                case LAND:
                    lhs = new ExprOrValue(lhs.assumeExpression(tok).and(parse_featureExprOrValue(pri, true).assumeExpression(tok)));// (lhs != 0) && (rhs
                    // != 0) ? 1 : 0;
                    break;
                case LOR:
                    lhs = new ExprOrValue(lhs.assumeExpression(tok).or(parse_featureExprOrValue(pri, true).assumeExpression(tok)));// (lhs != 0) || (rhs
                    // != 0) ? 1 : 0;
                    break;


                case '?':
                    lhs = parse_qifExpr(lhs.assumeExpression(tok), tok);
                    break;


                default:
                    error(op, "Unexpected operator " + op.getText());
                    return new ExprOrValue(FeatureExprLib.False(), FeatureExprLib.zero());


            }
        }

View Full Code Here

    }


    private FeatureExpr parse_definedExpr(boolean referToExternalDefinitionsOnly)
            throws IOException, LexerException {
        FeatureExpr lhs;
        Token la = source_token_nonwhite();
        boolean paren = false;
        if (la.getType() == '(') {
            paren = true;
            la = source_token_nonwhite();
        }


        // System.out.println("Core token is " + la);


        if (la.getType() != IDENTIFIER) {
            error(la, "defined() needs identifier, not " + getTextOrDefault(la, "<Feature Expression>"));
            lhs = FeatureExprLib.False();
        } else
            // System.out.println("Found macro");
            if (!(la.getSource().isNormalizedExternalFeatureExpr() || referToExternalDefinitionsOnly))
                lhs = FeatureExprLib.l().createDefinedMacro(la.getText(), macros);
            else
                /*
                 * when expression was created by expanding a macro (or by reading
                 * definedEx instead of defined), do not look up macro definition,
                 * it is already based on external features only
                 */
                lhs = FeatureExprLib.l().createDefinedExternal(la.getText());


        if (paren) {
            la = source_token_nonwhite();
            if (la.getType() != ')') {
                expr_untoken(la);
                error(la, "Missing ) in defined()");
            }
        }
        return lhs;

View Full Code Here

            return new ExprOrValue(FeatureExprLib.l().createIf(condition, thenBranch.assumeValue(tok), elseBranch.assumeValue(tok)));
    }


    private void consumeToken(int tokenType, boolean inlineCppExpression)
            throws IOException, LexerException {
        Token la = expr_token(inlineCppExpression);
        if (la.getType() != tokenType)
            error(la, "expected " + Token.getTokenName(tokenType)
                    + " but found " + Token.getTokenName(la.getType()));
    }

View Full Code Here

0 1 2

TOP

Related Classes of de.fosd.typechef.lexer.Token

de.fosd.typechef.lexer.CppReader

de.fosd.typechef.lexer.LexerSource

de.fosd.typechef.lexer.LexerSourceTestCase

de.fosd.typechef.lexer.Preprocessor

de.fosd.typechef.lexer.PreprocessorTestCase

de.fosd.typechef.lexer.Source

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.