Package org.python.pydev.parser.jython

Examples of org.python.pydev.parser.jython.Token


                IRegion endLine = doc.getLineInformationOfOffset(doc.getLength());
                errorStart = endLine.getOffset();
                errorEnd = endLine.getOffset() + endLine.getLength();

            } else {
                Token errorToken = parseErr.currentToken.next != null ? parseErr.currentToken.next
                        : parseErr.currentToken;
                IRegion startLine = doc.getLineInformation(getDocPosFromAstPos(errorToken.beginLine));
                IRegion endLine;
                if (errorToken.endLine == 0) {
                    endLine = startLine;
View Full Code Here


     * This is where we do a lookahead to see if we find some token and if we do find it, but not on the correct
     * position, we skip some tokens to go to it.
     */
    public ISpecialStr createSpecialStr(String token, boolean searchOnLast, boolean throwException)
            throws ParseException {
        final Token currentToken = grammar.getCurrentToken();

        Token firstTokenToIterate;
        if (searchOnLast) {
            firstTokenToIterate = grammar.getJJLastPos();
        } else {
            firstTokenToIterate = currentToken;
        }
        Token foundToken = null;

        int foundAtPos = 0;

        //lot's of tokens, but we'll bail out on an indent, or dedent, so, that's OK.
        TokensIterator iterTokens = grammar.getTokensIterator(firstTokenToIterate, 50, true);
        while (iterTokens.hasNext()) {
            foundAtPos += 1;
            Token next = iterTokens.next();
            if (next.image != null && next.image.equals(token)) {
                //Found what we were looking for!
                foundToken = next;
                break;
            }
        }

        if (foundToken != null) {
            if (foundAtPos <= 2 //found at correct position.
                    || searchOnLast //we already matched it... right now we're just adding it to the stack!
            ) {
                return foundToken.asSpecialStr();

            }
        }

        if (throwException) {
            ParseException e = createException(token, currentToken);

            //we found it at the wrong position!
            if (foundToken != null) {
                //we found it, but not on the position we were expecting, so, we must skip some tokens to get to it --
                //and report the needed exception)
                grammar.addAndReport(e, "Found at wrong position: " + foundToken);
                Token beforeLastReturned = iterTokens.getBeforeLastReturned();
                grammar.setCurrentToken(beforeLastReturned);
                return foundToken.asSpecialStr();
            }

            //create a 'synthetic token' in the place we were expecting it.
View Full Code Here

        return this.prev;
    }

    private void addSpecial(SimpleNode node, Object special, boolean after) {
        if (special instanceof Token) {
            Token t = (Token) special;
            if (t.toString().trim().startsWith("#")) {
                commentType comment = new commentType(t.image.trim());
                comment.beginColumn = t.beginColumn;
                comment.beginLine = t.beginLine;
                special = comment;

                if (node.beginLine != comment.beginLine) {
                    if (lastSpecial != null && lastNodeWithSpecial != null) {
                        if (comment.beginLine < lastSpecial.getBeginLine()
                                || (comment.beginLine == lastSpecial.getBeginLine() && comment.beginColumn < lastSpecial
                                        .getBeginCol())) {
                            List<Object> specialsBefore = lastNodeWithSpecial.getSpecialsBefore();
                            specialsBefore.add(specialsBefore.indexOf(lastSpecial), comment);
                            return;
                        }
                    }
                }
            } else {
                special = t.asSpecialStr();
            }
        }

        node.addSpecial(special, after);
    }
View Full Code Here

    private void handleFutureImports(String importName) {

    }

    protected final void addSpecialToArgDef(Object str) {
        Token token = (Token) str;
        SimpleNode peeked = jjtree.peekNode();
        if (peeked instanceof JfpDef) {
            JfpDef jfpdef = (JfpDef) peeked;
            if (jfpdef.typeDef != null) {
                jfpdef.typeDef.getSpecialsAfter().add(token.asSpecialStr());
            } else {
                jfpdef.nameNode.getSpecialsAfter().add(token.asSpecialStr());
            }
        } else if (peeked != null) {
            peeked.getSpecialsAfter().add(token.asSpecialStr());
        }

    }
View Full Code Here

    }

    //small_stmt: expr_stmt | del_stmt | pass_stmt | flow_stmt | import_stmt | global_stmt | nonlocal_stmt | assert_stmt
    final public void small_stmt() throws ParseException {
        SimpleNode simpleNode;
        Token spStr;
        switch ((jj_ntk == -1) ? jj_ntk() : jj_ntk) {
            case LPAREN:
            case LBRACE:
            case LBRACKET:
            case DOT:
View Full Code Here

        /*@bgen(jjtree) pass_stmt */
        SimpleNode jjtn000 = builder.openNode(JJTPASS_STMT);
        boolean jjtc000 = true;
        jjtree.openNodeScope(jjtn000);
        jjtreeOpenNodeScope(jjtn000);
        Token spStr;
        try {
            spStr = jj_consume_token(PASS);
            jjtree.closeNodeScope(jjtn000, true);
            jjtc000 = false;
            jjtreeCloseNodeScope(jjtn000);
View Full Code Here

        /*@bgen(jjtree) yield_expr */
        SimpleNode jjtn000 = builder.openNode(JJTYIELD_EXPR);
        boolean jjtc000 = true;
        jjtree.openNodeScope(jjtn000);
        jjtreeOpenNodeScope(jjtn000);
        Token spStr;
        boolean isYieldFrom = false;
        try {
            spStr = jj_consume_token(YIELD);
            switch ((jj_ntk == -1) ? jj_ntk() : jj_ntk) {
                case LPAREN:
View Full Code Here

        /*@bgen(jjtree) dotted_name */
        SimpleNode jjtn000 = builder.openNode(JJTDOTTED_NAME);
        boolean jjtc000 = true;
        jjtree.openNodeScope(jjtn000);
        jjtreeOpenNodeScope(jjtn000);
        Token t;
        FastStringBuffer sb = dottedNameStringBuffer.clear();
        try {
            t = Name();
            sb.append(t.image);
            label_16: while (true) {
View Full Code Here

        /*@bgen(jjtree) import_as_name */
        SimpleNode jjtn000 = builder.openNode(JJTIMPORT_AS_NAME);
        boolean jjtc000 = true;
        jjtree.openNodeScope(jjtn000);
        jjtreeOpenNodeScope(jjtn000);
        Token t;
        try {
            t = Name();
            switch ((jj_ntk == -1) ? jj_ntk() : jj_ntk) {
                case AS:
                    grammarActions.findTokenAndAdd("as");
View Full Code Here

        /*@bgen(jjtree) classdef */
        SimpleNode jjtn000 = builder.openNode(JJTCLASSDEF);
        boolean jjtc000 = true;
        jjtree.openNodeScope(jjtn000);
        jjtreeOpenNodeScope(jjtn000);
        Token spStr;
        Token spStr2;
        try {
            jj_consume_token(CLASS);
            this.markLastAsSuiteStart();
            Name();
            switch ((jj_ntk == -1) ? jj_ntk() : jj_ntk) {
View Full Code Here

TOP

Related Classes of org.python.pydev.parser.jython.Token

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.