Package org.apache.pig.impl.logicalLayer.schema.Schema

Examples of org.apache.pig.impl.logicalLayer.schema.Schema.FieldSchema

1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
        Schema expectedSch =
            Schema.generateNestedSchema(DataType.BAG, DataType.INTEGER);

        //check schema of TOBAG when given input tuple having only integers
        Schema inputSch = new Schema();
        inputSch.add(new FieldSchema(null, DataType.INTEGER));
        assertEquals("schema of tobag when input has only ints",
                expectedSch, tb.outputSchema(inputSch));

        //add another int column
        inputSch.add(new FieldSchema(null, DataType.INTEGER));
        assertEquals("schema of tobag when input has only ints",
                expectedSch, tb.outputSchema(inputSch));

        //add a long column
        inputSch.add(new FieldSchema(null, DataType.LONG));
        //expect null inner schema
        expectedSch =
            Schema.generateNestedSchema(DataType.BAG, DataType.NULL);
        assertEquals("schema of tobag when input has ints and long",
                expectedSch, tb.outputSchema(inputSch));

       
        //test schema when input is a tuple with inner schema
        Schema tupInSchema = new Schema(new FieldSchema("x", DataType.CHARARRAY));
        inputSch = new Schema();
        inputSch.add(new FieldSchema("a", tupInSchema, DataType.TUPLE));
        Schema inputSchCp = new Schema(inputSch);
        inputSchCp.getField(0).alias = null;
        expectedSch = new Schema(new FieldSchema(null, inputSchCp, DataType.BAG));
        assertEquals("schema of tobag when input has cols of type tuple ",
                expectedSch, tb.outputSchema(inputSch));
       
        inputSch.add(new FieldSchema("b", tupInSchema, DataType.TUPLE));
        assertEquals("schema of tobag when input has cols of type tuple ",
                expectedSch, tb.outputSchema(inputSch));
       
        //add a column of type tuple with different inner schema
        tupInSchema = new Schema(new FieldSchema("x", DataType.BYTEARRAY));
        inputSch.add(new FieldSchema("c", tupInSchema, DataType.TUPLE));
        //expect null inner schema
        expectedSch =
            Schema.generateNestedSchema(DataType.BAG, DataType.NULL);
        assertEquals("schema of tobag when input has cols of type tuple with diff inner schema",
                expectedSch, tb.outputSchema(inputSch));
View Full Code Here
78
79
80
81
82
83
84
85
86
87
88
89
90
91
        tschema.getColumn(col);
      if (columnSchema != null) {
        ColumnType ct = columnSchema.getType();
        if (ct == org.apache.hadoop.zebra.schema.ColumnType.RECORD ||
            ct == org.apache.hadoop.zebra.schema.ColumnType.COLLECTION)
          ret.add(new FieldSchema(col, toPigSchema(columnSchema.getSchema()), ct.pigDataType()));
        else
          ret.add(new FieldSchema(col, ct.pigDataType()));
      } else {
        ret.add(new FieldSchema(null, null));
      }
    }
    return ret;
  }
View Full Code Here
92
93
94
95
96
97
98
99
100
101
102
            description = "autogenerated from Pig Field Schema";
            Schema inner = fieldSchema.schema;
            if (type == DataType.BAG && fieldSchema.schema != null
                    && !fieldSchema.schema.isTwoLevelAccessRequired()) {
                log.info("Insert two-level access to Resource Schema");
                FieldSchema fs = new FieldSchema("t", fieldSchema.schema);
                inner = new Schema(fs);               
            }
           
            // allow partial schema
            if ((type == DataType.BAG || type == DataType.TUPLE)
View Full Code Here
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
                    int pos = (Integer)((LOConst)getArguments().get(0)).getValue();
                    LogicalOperator inp = implicitReferencedOperator;
                    if(inp.getSchema() != null){
                        // input logical operator has schema, copy and link
                        //to corresponding FieldSchema
                        FieldSchema inpFs = inp.getSchema().getField(pos);
                        mFieldSchema = Schema.FieldSchema.copyAndLink(inpFs, inp);
                    }else{
                        // no schema for input logicaloperator, use bytearray
                        // and set it as parent
                        mFieldSchema = new FieldSchema(null, DataType.BYTEARRAY);
                        mFieldSchema.setParent(null, inp);
                    }
                    mIsFieldSchemaComputed = true;
                    return mFieldSchema;
                }else{
View Full Code Here
412
413
414
415
416
417
418
419
420
421
422
  List<FieldSchema> fieldSchemaList = new ArrayList<FieldSchema>(
    cols.size());

  for (int i = 0; i < cols.size(); i++) {
      fieldSchemaList.add(new FieldSchema(cols.get(i), HiveRCSchemaUtil
        .findPigDataType(types.get(i))));
  }

  pigSchema = new ResourceSchema(new Schema(fieldSchemaList));
View Full Code Here
660
661
662
663
664
665
666
667
668
669
670
    int index = fields.length;

    for (String key : keys) {
        newFields[index++] = new ResourceFieldSchema(
          new FieldSchema(key, DataType.CHARARRAY));
    }

    pigSchema.setFields(newFields);

    LOG.debug("Added partition fields: " + keys
View Full Code Here
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
                    Schema cSchema = op.getSchema();
                    if(cSchema!=null){
                       
                        for (FieldSchema schema : cSchema.getFields()) {
                            ++i;
                            FieldSchema newFS = null;
                            if(schema.alias != null) {
                                if(nonDuplicates.containsKey(schema.alias)) {
                                    if (nonDuplicates.get(schema.alias) != -1) {
                                        nonDuplicates.remove(schema.alias);
                                        nonDuplicates.put(schema.alias, -1);
                                    }
                                } else {
                                    nonDuplicates.put(schema.alias, i);
                                }
                                newFS = new FieldSchema(op.getAlias()+"::"+schema.alias,schema.schema,schema.type);
                            } else {
                                newFS = new Schema.FieldSchema(null, schema.type);
                            }
                            newFS.setParent(schema.canonicalName, op);
                            fss.add(newFS);
                            mSchemaInputMapping.add(op);
                        }
                    } else {
                        seeUnknown = true;
                    }
                } catch (FrontendException ioe) {
                    mIsSchemaComputed = false;
                    mSchema = null;
                    throw ioe;
                }
            }
            mIsSchemaComputed = true;
            mSchema = null;
            if (!seeUnknown)
            {
                mSchema = new Schema(fss);
                for (Entry<String, Integer> ent : nonDuplicates.entrySet()) {
                    int ind = ent.getValue();
                    if(ind==-1) continue;
                    FieldSchema prevSch = fss.get(ind);
                    // this is a non duplicate and hence can be referred to
                    // with just the field schema alias or outeralias::field schema alias
                    // In mSchema we have outeralias::fieldschemaalias. To allow
                    // using just the field schema alias, add it to mSchemas
                    // as an alias for this field.
View Full Code Here
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
     * @throws VisitorException
     */
    private boolean byteArrayFound(Schema s) throws VisitorException {
        for(int i=0;i<s.size();i++){
            try {
                FieldSchema fs=s.getField(i);
                if(fs.type==DataType.BYTEARRAY){
                    return true;
                }
            } catch (FrontendException fee) {
                int errCode = 1043;
View Full Code Here
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
    private List<Integer> getByteArrayPositions(Schema s)
            throws VisitorException {
        List<Integer> result = new ArrayList<Integer>();
        for (int i = 0; i < s.size(); i++) {
            try {
                FieldSchema fs = s.getField(i);
                if (fs.type == DataType.BYTEARRAY) {
                    result.add(i);
                }
            } catch (FrontendException fee) {
                int errCode = 1043;
View Full Code Here
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
        Iterator<FieldSchema> i = inputSchema.getFields().iterator();
        Iterator<FieldSchema> j = udfSchema.getFields().iterator();

        while (i.hasNext()) {

            FieldSchema inputFieldSchema = i.next();
            FieldSchema udfFieldSchema = j.next();

            if(ignoreByteArrays && inputFieldSchema.type == DataType.BYTEARRAY) {
                continue;
            }
           
View Full Code Here
TOP

Related Classes of org.apache.pig.impl.logicalLayer.schema.Schema.FieldSchema

  • com.acme.math.Pow
  • com.cloudera.science.pig.Bin
  • com.cloudera.science.pig.Combinatorial
  • com.cloudera.science.pig.Quantile
  • com.mozilla.pig.eval.Coalesce
  • com.twitter.elephantbird.pig.load.LocationAsTuple
  • com.twitter.elephantbird.pig.load.LzoW3CLogLoader
  • com.twitter.elephantbird.pig.mahout.VectorWritableConverter
  • com.twitter.elephantbird.pig.piggybank.GenericInvoker
  • com.twitter.elephantbird.pig.util.ProtobufToPig
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.