Package org.apache.pig.newplan.logical.relational.LogicalSchema

Examples of org.apache.pig.newplan.logical.relational.LogicalSchema.LogicalFieldSchema


                boolean needNewUid = innerLoadsPair.second;

                // pull tuple information from innerload
                if (innerLoads.get(0).getProjection().getFieldSchema().schema!=null &&
                        innerLoads.get(0).getProjection().getFieldSchema().type==DataType.BAG) {
                    LogicalFieldSchema originalTupleFieldSchema = innerLoads.get(0).getProjection().getFieldSchema().schema.getField(0);
                    LogicalFieldSchema newTupleFieldSchema = new LogicalFieldSchema(originalTupleFieldSchema.alias,
                            schema, DataType.TUPLE);
                    if (needNewUid) {
                        newTupleFieldSchema.uid = LogicalExpression.getNextUid();
                    }
                    else {
                        newTupleFieldSchema.uid = originalTupleFieldSchema.uid;
                    }
                    LogicalSchema newTupleSchema = new LogicalSchema();
                    newTupleSchema.addField(newTupleFieldSchema);
                    fieldSchema = new LogicalSchema.LogicalFieldSchema(alias, newTupleSchema, DataType.BAG);
                }
                else {
                    fieldSchema = new LogicalSchema.LogicalFieldSchema(alias, schema, DataType.BAG);
                }
                if (needNewUid)
                    fieldSchema.uid = LogicalExpression.getNextUid();
                else
                    fieldSchema.uid = innerLoads.get(0).getProjection().getFieldSchema().uid;
            }
            else {
                // InnerLoad and source is not bag
                if(schema == null){
                    // if we get here, it is range or starProject, otherwise, innerLoad will convert schema to non-null
                    if (isRangeProject && endCol!=-1){
                        LogicalSchema innerSchema = new LogicalSchema();
                        for(int i = startCol; i <= endCol; i++){
                            //schema is null, so null alias
                            innerSchema.addField(new LogicalFieldSchema(null, null, DataType.BYTEARRAY));
                        }
                        fieldSchema = new LogicalSchema.LogicalFieldSchema(null, innerSchema, DataType.TUPLE);
                    } else {
                        fieldSchema = null;
                    }
                }
                else{
                    fieldSchema = schema.getField(0);
                }
            }
            if (fieldSchema!=null)
                uidOnlyFieldSchema = fieldSchema.mergeUid(uidOnlyFieldSchema);
        }
        else {
            if (schema == null) {
                if(isRangeOrStarProject()) {
                    if (isRangeProject && endCol!=-1){
                        LogicalSchema innerSchema = new LogicalSchema();
                        for(int i = startCol; i <= endCol; i++){
                            //schema is null, so null alias
                            innerSchema.addField(new LogicalFieldSchema(null, null, DataType.BYTEARRAY));
                        }
                        fieldSchema = new LogicalSchema.LogicalFieldSchema(null, innerSchema, DataType.TUPLE);
                    }
                    else {
                        fieldSchema = null;
                    }
                } else {
                    fieldSchema = new LogicalSchema.LogicalFieldSchema(null, null, DataType.BYTEARRAY);
                }

                if (fieldSchema!=null)
                    uidOnlyFieldSchema = fieldSchema.mergeUid(uidOnlyFieldSchema);
            }
            else {
                int index = -1;
                if (!isRangeOrStarProject() && uidOnlyFieldSchema!=null) {
                    long uid = uidOnlyFieldSchema.uid;
                    for (int i=0;i<schema.size();i++) {
                        LogicalFieldSchema fs = schema.getField(i);
                        if (fs.uid==uid) {
                            index = i;
                        }
                    }
                }
View Full Code Here


    @Override
    public LogicalSchema.LogicalFieldSchema getFieldSchema() throws FrontendException {
        if (fieldSchema!=null)
            return fieldSchema;
        LogicalExpression successor = (LogicalExpression)plan.getSuccessors(this).get(0);
        LogicalFieldSchema predFS = successor.getFieldSchema();
        if (predFS!=null) {
            if (predFS.type==DataType.BAG) {
                LogicalSchema innerSchema = null;
                if (predFS.schema!=null) {
                    innerSchema = new LogicalSchema();
                    // Get the tuple inner schema
                    LogicalSchema origSchema = predFS.schema.getField(0).schema;;
                    // Slice the tuple inner schema
                    if (!rawColumns.isEmpty()) {
                        columns = translateAliasToPos(origSchema, rawColumns);
                    }
                    for (int column:columns) {
                        if (origSchema!=null && origSchema.size()!=0) {
                            innerSchema.addField(origSchema.getField(column));
                        }
                        else {
                            innerSchema.addField(new LogicalFieldSchema(null, null, DataType.BYTEARRAY));
                        }
                    }
                }
                LogicalSchema bagSchema = new LogicalSchema();
                bagSchema.addField(new LogicalSchema.LogicalFieldSchema(null, innerSchema, DataType.TUPLE,
                        LogicalExpression.getNextUid()));
                fieldSchema = new LogicalSchema.LogicalFieldSchema(null, bagSchema, DataType.BAG, LogicalExpression.getNextUid());
                uidOnlyFieldSchema = fieldSchema.mergeUid(uidOnlyFieldSchema);
            }
            else { // Dereference a field out of a tuple
                if (predFS.schema!=null) {
                    if (!rawColumns.isEmpty()) {
                        columns = translateAliasToPos(predFS.schema, rawColumns);
                    }
                    if (predFS.schema!=null && predFS.schema.size()!=0) {
                        if (columns.size()==1) {
                            fieldSchema = predFS.schema.getField(columns.get(0));
                        } else {
                            LogicalSchema innerSchema = new LogicalSchema();
                            String alias = predFS.alias;
                            for (int column:columns) {
                                innerSchema.addField(predFS.schema.getField(column));
                                String subAlias = predFS.schema.getField(column).alias;
                                if (subAlias==null) {
                                    subAlias = "";
                                }
                                alias = alias + "_" + subAlias;
                            }
                            fieldSchema = new LogicalSchema.LogicalFieldSchema(alias, innerSchema, DataType.TUPLE, LogicalExpression.getNextUid());
                        }
                    }
                    else {
                        fieldSchema = new LogicalSchema.LogicalFieldSchema(null, null, DataType.BYTEARRAY);
                        uidOnlyFieldSchema = fieldSchema.mergeUid(uidOnlyFieldSchema);
                    }
                } else{
                    fieldSchema = new LogicalFieldSchema(null, null, DataType.BYTEARRAY);
                    uidOnlyFieldSchema = fieldSchema.mergeUid(uidOnlyFieldSchema);
                }
            }
        }
        return fieldSchema;
View Full Code Here

    public LogicalSchema.LogicalFieldSchema getFieldSchema() throws FrontendException {
        if (fieldSchema!=null)
            return fieldSchema;

        //TypeCheckingExpVisitor will ensure that lhs and rhs have same schema
        LogicalFieldSchema argFs = getLhs().getFieldSchema();
        fieldSchema = argFs.deepCopy();
        fieldSchema.resetUid();

        uidOnlyFieldSchema = fieldSchema.mergeUid(uidOnlyFieldSchema);
        return fieldSchema;
    }
View Full Code Here

     * @param exprPlan ExpressionPlan which generates this field
     * @return
     */
    private LogicalFieldSchema getPlanSchema( LogicalExpressionPlan exprPlan ) throws FrontendException {
        LogicalExpression sourceExp = (LogicalExpression) exprPlan.getSources().get(0);
        LogicalFieldSchema planSchema = null;
        if (sourceExp.getFieldSchema()!=null)
            planSchema = sourceExp.getFieldSchema().deepCopy();
        return planSchema;
    }
View Full Code Here

                hasMultipleKeys = true;
                break;
            }
        }

        LogicalFieldSchema groupKeySchema = null;
        // Generate the groupField Schema
        if( hasMultipleKeys ) {
            LogicalSchema keySchema = new LogicalSchema();
            // We sort here to maintain the correct order of inputs
            for( Integer key : mExpressionPlans.keySet()) {
                Collection<LogicalExpressionPlan> plans =
                    mExpressionPlans.get(key);

                for( LogicalExpressionPlan plan : plans ) {
                    LogicalFieldSchema fieldSchema = getPlanSchema(plan);
                    // if any plan schema is null, that means we can't calculate
                    // further schemas so we bail out
                    if( fieldSchema == null ) {
                        schema = null;
                        return schema;
                    }
                    fieldSchema = new LogicalFieldSchema(fieldSchema);
                    keySchema.addField(fieldSchema);
                }
                // We only need fields from one input and not all
                break;
            }
            groupKeySchema = new LogicalFieldSchema(GROUP_COL_NAME, keySchema, DataType.TUPLE);
        } else {
            // We sort here to maintain the correct order of inputs
            for( Integer key : mExpressionPlans.keySet() ) {
                Collection<LogicalExpressionPlan> plans = mExpressionPlans.get(key);
                for( LogicalExpressionPlan plan : plans ) {
                    groupKeySchema = getPlanSchema(plan);
                    // if any plan schema is null, that means we cannot figure out
                    // the arity of keys, just give an empty tuple
                    if( groupKeySchema == null ) {
                        groupKeySchema = new LogicalSchema.LogicalFieldSchema("group", null, DataType.TUPLE);
                        break;
                    }
                    groupKeySchema = new LogicalSchema.LogicalFieldSchema(groupKeySchema);
                    // Change the uid of this field
                    groupKeySchema.alias = GROUP_COL_NAME;
                    break;
                }
                break;
            }          
        }
        if(mExpressionPlans.size() > 1){
            //reset the uid, because the group column is associated with more
            // than one input
            groupKeySchema.resetUid();
        }
       
        if (groupKeySchema==null) {
            throw new FrontendException(this, "Cannot get group key schema for " + this, 2234);
        }
        groupKeyUidOnlySchema = groupKeySchema.mergeUid(groupKeyUidOnlySchema);

        fieldSchemaList.add( groupKeySchema );

        // Generate the Bag Schema
        int counter = 0;
        for (Operator op : inputs) {
            LogicalSchema inputSchema = ((LogicalRelationalOperator)op).getSchema();
          
            // Check if we already have calculated Uid for this bag for given
            // input operator
            long bagUid;
            if (generatedInputUids.get(counter)!=null)
                bagUid = generatedInputUids.get(counter);
            else {
                bagUid = LogicalExpression.getNextUid();
                generatedInputUids.put( counter, bagUid );
            }
           
            LogicalFieldSchema newTupleFieldSchema = new LogicalFieldSchema(
                    null, inputSchema, DataType.TUPLE, LogicalExpression.getNextUid());
           
            LogicalSchema bagSchema = new LogicalSchema();
            bagSchema.addField(newTupleFieldSchema);
           
            LogicalFieldSchema newBagFieldSchema = new LogicalFieldSchema(
                    ((LogicalRelationalOperator)op).getAlias(), bagSchema,
                    DataType.BAG, bagUid);

            fieldSchemaList.add( newBagFieldSchema );
            counter ++;
View Full Code Here

            }
            if (uid==-1) {
                uid = LogicalExpression.getNextUid();
                for (Operator input : inputs) {
                    long inputUid;
                    LogicalFieldSchema matchedInputFieldSchema;
                  if (onSchema) {
                      matchedInputFieldSchema = ((LogicalRelationalOperator)input).getSchema().getFieldSubNameMatch(s0.getField(i).alias);
                        if (matchedInputFieldSchema!=null) {
                            inputUid = matchedInputFieldSchema.uid;
                            uidMapping.add(new Pair<Long, Long>(uid, inputUid));
View Full Code Here

    protected LogicalSchema determineSchema(LogicalRelationalOperator op)
            throws FrontendException {
        LogicalSchema determinedSchema = new LogicalSchema();
        for (int i = 0; i < op.getSchema().size(); i++) {
            determinedSchema.addField(new LogicalFieldSchema(
                    null,
                    null,
                    DataType.BYTEARRAY));
        }
        return determinedSchema;
View Full Code Here

        @Override
        public void visit(LOLoad load) throws FrontendException {
            if( load.getSchema() != null ) {
                Map<Integer,Set<String>> annotation = new HashMap<Integer,Set<String>>();
                for( int i=0; i<load.getSchema().size(); i++) {
                    LogicalFieldSchema field = load.getSchema().getField(i);
                    if( inputUids.containsKey( field.uid ) ) {
                        annotation.put(i, inputUids.get( field.uid ) );
                    }
                }
                load.annotate(REQUIRED_MAPKEYS, annotation);
View Full Code Here

                for (LogicalSchema.LogicalFieldSchema fs : mUserDefinedSchema.get(i).getFields()) {
                    mUserDefinedSchemaCopy.addField(fs.deepCopy());
                }
            }
           
            LogicalFieldSchema fieldSchema = null;
           
            // schema of the expression after flatten
            LogicalSchema expSchema = null;
           
            if (exp.getFieldSchema()!=null) {
           
                fieldSchema = exp.getFieldSchema().deepCopy();
               
                expSchema = new LogicalSchema();
                if ((fieldSchema.type != DataType.TUPLE && fieldSchema.type != DataType.BAG)||!flattenFlags[i]) {
                    // if type is primitive, just add to schema
                    if (fieldSchema!=null)
                        expSchema.addField(fieldSchema);
                    else
                        expSchema = null;
                } else {
                    // if bag/tuple don't have inner schema, after flatten, we don't have schema for the entire operator
                    if (fieldSchema.schema==null) {
                        expSchema = null;
                    }
                    else {
                     // if we come here, we get a BAG/Tuple with flatten, extract inner schema of the tuple as expSchema
                        List<LogicalSchema.LogicalFieldSchema> innerFieldSchemas = new ArrayList<LogicalSchema.LogicalFieldSchema>();
                        if (flattenFlags[i]) {
                            if (fieldSchema.type == DataType.BAG) {
                                // if it is bag, get the schema of tuples
                                if (fieldSchema.schema!=null) {
                                    if (fieldSchema.schema.getField(0).schema!=null)
                                        innerFieldSchemas = fieldSchema.schema.getField(0).schema.getFields();
                                    for (LogicalSchema.LogicalFieldSchema fs : innerFieldSchemas) {
                                        fs.alias = fs.alias == null ? null : fieldSchema.alias + "::" + fs.alias;
                                    }
                                }
                            } else { // DataType.TUPLE
                                innerFieldSchemas = fieldSchema.schema.getFields();
                                for (LogicalSchema.LogicalFieldSchema fs : innerFieldSchemas) {
                                    fs.alias = fs.alias == null ? null : fieldSchema.alias + "::" + fs.alias;
                                }
                            }
                           
                            for (LogicalSchema.LogicalFieldSchema fs : innerFieldSchemas)
                                expSchema.addField(fs);
                        }
                        else
                            expSchema.addField(fieldSchema);
                    }
                }
            }
           
            // Merge with user defined schema
            if (expSchema!=null && expSchema.size()==0)
                expSchema = null;
            LogicalSchema planSchema = new LogicalSchema();
            if (mUserDefinedSchemaCopy!=null) {
                LogicalSchema mergedSchema = new LogicalSchema();
                // merge with userDefinedSchema
                if (expSchema==null) {
                    // Use user defined schema
                    for (LogicalFieldSchema fs : mUserDefinedSchemaCopy.getFields()) {
                        fs.stampFieldSchema();
                        mergedSchema.addField(new LogicalFieldSchema(fs));
                    }
                    for (LogicalFieldSchema fs : mergedSchema.getFields()) {
                        if (fs.type == DataType.NULL){
                            //this is the use case where a new alias has been specified by user
                            fs.type = DataType.BYTEARRAY;
View Full Code Here

                // add a logical schema for dimensions that are pushed from
                // predecessor of cube/rollup
                if (oper instanceof ProjectExpression) {
                    LogicalSchema output = new LogicalSchema();
                    output.addField(new LogicalFieldSchema(
                        ((ProjectExpression) oper).getColAlias(), null, DataType.NULL));
                    genOutputSchema.add(output);
                } else if (oper instanceof UserFuncExpression) {
                    // add logical schema for dimensions specified in
                    // cube/rollup operator
                    LogicalSchema output = new LogicalSchema();
                    for (Operator op : ((UserFuncExpression) oper).getPlan().getSinks()) {
                        output.addField(new LogicalFieldSchema(((ProjectExpression) op)
                                .getFieldSchema()));
                    }
                    genOutputSchema.add(output);
                }
View Full Code Here

TOP

Related Classes of org.apache.pig.newplan.logical.relational.LogicalSchema.LogicalFieldSchema

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.