Package org.apache.pig.newplan.logical.relational.LogicalSchema

Examples of org.apache.pig.newplan.logical.relational.LogicalSchema.LogicalFieldSchema

445
446
447
448
449
450
451
452
     * @param arg
     * @throws FrontendException
     */
    private void insertCast(LogicalExpression exp, byte toType, LogicalExpression arg)
    throws FrontendException {
        LogicalFieldSchema toFs = new LogicalSchema.LogicalFieldSchema(null, null, toType);
        insertCast(exp, toFs, arg);
    }
View Full Code Here
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
            String msg = "Cannot cast to bytearray";
            msgCollector.collect(msg, MessageType.Error) ;
            throw new TypeCheckerException(cast, msg, errCode, PigException.INPUT) ;
        }

        LogicalFieldSchema inFs = cast.getExpression().getFieldSchema();
        LogicalFieldSchema outFs = cast.getFieldSchema();

        if(inFs == null){
            //replace null schema with bytearray schema.
            inFs = new LogicalFieldSchema(null, null, DataType.BYTEARRAY);
        }

        //check if the field schemas are castable
        boolean castable = LogicalFieldSchema.castable(inFs, outFs);
        if(!castable) {
            int errCode = 1052;
            String msg = "Cannot cast "
                           + DataType.findTypeName(inType)
                           + ((DataType.isSchemaType(inType))? " with schema " + inFs.toString(false) : "")
                           + " to "
                           + DataType.findTypeName(outType)
                           + ((DataType.isSchemaType(outType))? " with schema " + outFs.toString(false) : "");
            msgCollector.collect(msg, MessageType.Error) ;
            throw new TypeCheckerException(cast, msg, errCode, PigException.INPUT) ;
        }

    }
View Full Code Here
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
                boolean needNewUid = innerLoadsPair.second;

                // pull tuple information from innerload
                if (innerLoads.get(0).getProjection().getFieldSchema().schema!=null &&
                        innerLoads.get(0).getProjection().getFieldSchema().type==DataType.BAG) {
                    LogicalFieldSchema originalTupleFieldSchema = innerLoads.get(0).getProjection().getFieldSchema().schema.getField(0);
                    LogicalFieldSchema newTupleFieldSchema = new LogicalFieldSchema(originalTupleFieldSchema.alias,
                            schema, DataType.TUPLE);
                    if (needNewUid) {
                        newTupleFieldSchema.uid = LogicalExpression.getNextUid();
                    }
                    else {
                        newTupleFieldSchema.uid = originalTupleFieldSchema.uid;
                    }
                    LogicalSchema newTupleSchema = new LogicalSchema();
                    newTupleSchema.addField(newTupleFieldSchema);
                    fieldSchema = new LogicalSchema.LogicalFieldSchema(alias, newTupleSchema, DataType.BAG);
                }
                else {
                    fieldSchema = new LogicalSchema.LogicalFieldSchema(alias, schema, DataType.BAG);
                }
                if (needNewUid)
                    fieldSchema.uid = LogicalExpression.getNextUid();
                else
                    fieldSchema.uid = innerLoads.get(0).getProjection().getFieldSchema().uid;
            }
            else {
                // InnerLoad and source is not bag
                if(schema == null){
                    // if we get here, it is range or starProject, otherwise, innerLoad will convert schema to non-null
                    if (isRangeProject && endCol!=-1){
                        LogicalSchema innerSchema = new LogicalSchema();
                        for(int i = startCol; i <= endCol; i++){
                            //schema is null, so null alias
                            innerSchema.addField(new LogicalFieldSchema(null, null, DataType.BYTEARRAY));
                        }
                        fieldSchema = new LogicalSchema.LogicalFieldSchema(null, innerSchema, DataType.TUPLE);
                    } else {
                        fieldSchema = null;
                    }
                }
                else{
                    fieldSchema = schema.getField(0);
                }
            }
            if (fieldSchema!=null)
                uidOnlyFieldSchema = fieldSchema.mergeUid(uidOnlyFieldSchema);
        }
        else {
            if (schema == null) {
                if(isRangeOrStarProject()) {
                    if (isRangeProject && endCol!=-1){
                        LogicalSchema innerSchema = new LogicalSchema();
                        for(int i = startCol; i <= endCol; i++){
                            //schema is null, so null alias
                            innerSchema.addField(new LogicalFieldSchema(null, null, DataType.BYTEARRAY));
                        }
                        fieldSchema = new LogicalSchema.LogicalFieldSchema(null, innerSchema, DataType.TUPLE);
                    }
                    else {
                        fieldSchema = null;
                    }
                } else {
                    fieldSchema = new LogicalSchema.LogicalFieldSchema(null, null, DataType.BYTEARRAY);
                }

                if (fieldSchema!=null)
                    uidOnlyFieldSchema = fieldSchema.mergeUid(uidOnlyFieldSchema);
            }
            else {
                int index = -1;
                if (!isRangeOrStarProject() && uidOnlyFieldSchema!=null) {
                    long uid = uidOnlyFieldSchema.uid;
                    for (int i=0;i<schema.size();i++) {
                        LogicalFieldSchema fs = schema.getField(i);
                        if (fs.uid==uid) {
                            index = i;
                        }
                    }
                }
View Full Code Here
88
89
90
91
92
93
94
95
96
97
98
        Util.optimizeNewLP(lp);
        LOStore loStore = (LOStore)lp.getSinks().get(0);
        LOForEach loForEach = (LOForEach)lp.getPredecessors(loStore).get(0);
        LogicalSchema schema = loForEach.getSchema();
        assertEquals(1, schema.size());
        LogicalFieldSchema bagFieldSchema = schema.getField(0);
        assertEquals(DataType.BAG, bagFieldSchema.type);
        LogicalFieldSchema tupleFieldSchema = bagFieldSchema.schema.getField(0);
        assertEquals(1, tupleFieldSchema.schema.size());
        assertEquals(DataType.BYTEARRAY, tupleFieldSchema.schema.getField(0).type);
    }
View Full Code Here
122
123
124
125
126
127
128
129
130
131
        Util.optimizeNewLP(lp);
        LOStore loStore = (LOStore)lp.getSinks().get(0);
        LOForEach loForEach = (LOForEach)lp.getPredecessors(loStore).get(0);
        LogicalSchema schema = loForEach.getSchema();
        assertEquals(1, schema.size());
        LogicalFieldSchema bagFieldSchema = schema.getField(0);
        assertEquals(DataType.BAG, bagFieldSchema.type);
        LogicalFieldSchema tupleFieldSchema = bagFieldSchema.schema.getField(0);
        assertNull(tupleFieldSchema.schema);
    }
View Full Code Here
139
140
141
142
143
144
145
146
147
148
149
        LogicalPlan lp = Util.parseAndPreprocess(query, pc);
        Util.optimizeNewLP(lp);
        LOStore loStore = (LOStore)lp.getSinks().get(0);
        LOSort loSort = (LOSort)lp.getPredecessors(loStore).get(0);
        Operator sortPlanLeaf = loSort.getSortColPlans().get(0).getSources().get(0);
        LogicalFieldSchema sortPlanFS = ((LogicalExpression)sortPlanLeaf).getFieldSchema();
        assertNull(sortPlanFS);

        PhysicalPlan pp = Util.buildPhysicalPlanFromNewLP(lp, pc);
        POStore poStore = (POStore)pp.getLeaves().get(0);
        POSort poSort = (POSort)pp.getPredecessors(poStore).get(0);
View Full Code Here
159
160
161
162
163
164
165
166
167
        LogicalPlan lp = Util.parseAndPreprocess(query, pc);
        Util.optimizeNewLP(lp);
        LOStore loStore = (LOStore)lp.getSinks().get(0);
        LOCogroup loCoGroup = (LOCogroup)lp.getPredecessors(loStore).get(0);
        LogicalFieldSchema groupFieldSchema = loCoGroup.getSchema().getField(0);
        assertEquals(DataType.TUPLE, groupFieldSchema.type);
        assertNull(groupFieldSchema.schema);
    }
View Full Code Here
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
                for (LogicalSchema.LogicalFieldSchema fs : mUserDefinedSchema.get(i).getFields()) {
                    mUserDefinedSchemaCopy.addField(fs.deepCopy());
                }
            }
           
            LogicalFieldSchema fieldSchema = null;
           
            // schema of the expression after flatten
            LogicalSchema expSchema = null;
           
            if (exp.getFieldSchema()!=null) {
           
                fieldSchema = exp.getFieldSchema().deepCopy();
               
                expSchema = new LogicalSchema();
                if ((fieldSchema.type != DataType.TUPLE && fieldSchema.type != DataType.BAG)||!flattenFlags[i]) {
                    // if type is primitive, just add to schema
                    if (fieldSchema!=null)
                        expSchema.addField(fieldSchema);
                    else
                        expSchema = null;
                } else {
                    // if bag/tuple don't have inner schema, after flatten, we don't have schema for the entire operator
                    if (fieldSchema.schema==null) {
                        expSchema = null;
                    }
                    else {
                     // if we come here, we get a BAG/Tuple with flatten, extract inner schema of the tuple as expSchema
                        List<LogicalSchema.LogicalFieldSchema> innerFieldSchemas = new ArrayList<LogicalSchema.LogicalFieldSchema>();
                        if (flattenFlags[i]) {
                            if (fieldSchema.type == DataType.BAG) {
                                // if it is bag, get the schema of tuples
                                if (fieldSchema.schema!=null) {
                                    if (fieldSchema.schema.getField(0).schema!=null)
                                        innerFieldSchemas = fieldSchema.schema.getField(0).schema.getFields();
                                    for (LogicalSchema.LogicalFieldSchema fs : innerFieldSchemas) {
                                        fs.alias = fs.alias == null ? null : fieldSchema.alias + "::" + fs.alias;
                                    }
                                }
                            } else { // DataType.TUPLE
                                innerFieldSchemas = fieldSchema.schema.getFields();
                                for (LogicalSchema.LogicalFieldSchema fs : innerFieldSchemas) {
                                    fs.alias = fs.alias == null ? null : fieldSchema.alias + "::" + fs.alias;
                                }
                            }
                           
                            for (LogicalSchema.LogicalFieldSchema fs : innerFieldSchemas)
                                expSchema.addField(fs);
                        }
                        else
                            expSchema.addField(fieldSchema);
                    }
                }
            }
           
            // Merge with user defined schema
            if (expSchema!=null && expSchema.size()==0)
                expSchema = null;
            LogicalSchema planSchema = new LogicalSchema();
            if (mUserDefinedSchemaCopy!=null) {
                LogicalSchema mergedSchema = new LogicalSchema();
                // merge with userDefinedSchema
                if (expSchema==null) {
                    // Use user defined schema
                    for (LogicalFieldSchema fs : mUserDefinedSchemaCopy.getFields()) {
                        fs.stampFieldSchema();
                        mergedSchema.addField(new LogicalFieldSchema(fs));
                    }
                    if(mergedSchema.size() == 1 && mergedSchema.getField(0).type == DataType.NULL){
                        //this is the use case where a new alias has been specified by user
                        mergedSchema.getField(0).type = DataType.BYTEARRAY;
                    }
View Full Code Here
340
341
342
343
344
345
346
347
348
349
350
                                    && userStarSch.getField(schIdx) != null){

                                //if the project-star field has user specified schema, use the
                                // j'th field for this column
                                LogicalSchema sch = new LogicalSchema();
                                sch.addField(new LogicalFieldSchema(userStarSch.getField(schIdx)));
                                newUserSchema.add(sch);
                            }
                            else{
                                newUserSchema.add(null);
                            }
View Full Code Here
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
        preds.addAll( plan.getPredecessors( union ) );

        List<LogicalSchema> fieldSchemas = new ArrayList<LogicalSchema>( fieldCount );
        for( LogicalFieldSchema fs : outputSchema.getFields() ) {
            LogicalSchema ls = new LogicalSchema();
            ls.addField( new LogicalFieldSchema( fs.alias, null, DataType.NULL ) );
            fieldSchemas.add( ls );
        }
       
        for( Operator pred : preds ) {
            LogicalRelationalOperator op = (LogicalRelationalOperator)pred;
            LogicalSchema opSchema = op.getSchema();
            if( opSchema.isEqual( outputSchema , true) )
                continue;
           
            LOForEach foreach = new LOForEach( plan );
            LogicalPlan innerPlan = new LogicalPlan();

            LOGenerate gen = new LOGenerate( innerPlan );
            boolean[] flattenFlags = new boolean[fieldCount];
            List<LogicalExpressionPlan> exprPlans = new ArrayList<LogicalExpressionPlan>( fieldCount );
            List<Operator> genInputs = new ArrayList<Operator>();
           
            // Get exprPlans, and genInputs
            for( LogicalFieldSchema fs : outputSchema.getFields() ) {
                LogicalExpressionPlan exprPlan = new LogicalExpressionPlan();
                exprPlans.add( exprPlan );
                int pos = -1;
                //do a match with subname also
                LogicalFieldSchema matchFS = opSchema.getFieldSubNameMatch(fs.alias);
                if(matchFS != null){
                    pos = opSchema.getFieldPosition(matchFS.alias);
                }
                if( pos == -1 ) {
                    ConstantExpression constExp = new ConstantExpression( exprPlan, null);
View Full Code Here
TOP

Related Classes of org.apache.pig.newplan.logical.relational.LogicalSchema.LogicalFieldSchema

  • org.apache.pig.newplan.logical.expression.BinCondExpression
  • org.apache.pig.newplan.logical.expression.DereferenceExpression
  • org.apache.pig.newplan.logical.expression.MapLookupExpression
  • org.apache.pig.newplan.logical.expression.ProjectExpression
  • org.apache.pig.newplan.logical.expression.ScalarExpression
  • org.apache.pig.newplan.logical.expression.UserFuncExpression
  • org.apache.pig.newplan.logical.relational.LOCogroup
  • org.apache.pig.newplan.logical.relational.LOGenerate
  • org.apache.pig.newplan.logical.relational.LOInnerLoad
  • org.apache.pig.newplan.logical.relational.LOUnion
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.