Examples of org.apache.pig.impl.io.NullablePartitionWritable

org.apache.pig.impl.io.NullablePartitionWritable
NullablePartitionWritable is an adaptor class around PigNullableWritable that adds a partition index to the class.

        pop.getPkgr().setInner(inner);
        PigNullableWritable k = HDataType.getWritableComparableTypes(key, keyType);
        pop.attachInput(k, db.iterator());
        if (keyType != DataType.BAG) {
            // test serialization
            NullablePartitionWritable wr;
            if (keyType == DataType.TUPLE) {
                BinSedesTuple tup = (BinSedesTuple) binfactory.newTupleNoCopy(((Tuple) k.getValueAsPigType()).getAll());
                wr = new NullablePartitionWritable(new NullableTuple(tup));
            } else {
                wr = new NullablePartitionWritable(k);
            }
            ByteArrayOutputStream baos = new ByteArrayOutputStream();
            DataOutputStream out = new DataOutputStream(baos);
            wr.write(out);
            byte[] arr = baos.toByteArray();
            ByteArrayInputStream bais = new ByteArrayInputStream(arr);
            DataInputStream in = new DataInputStream(bais);
            NullablePartitionWritable re = new NullablePartitionWritable();
            re.readFields(in);
            assertEquals(re, wr);
        }


        // we are not doing any optimization to remove
        // parts of the "value" which are present in the "key" in this

View Full Code Here


                PigNullableWritable key =
                        HDataType.getWritableComparableTypes(result.get(2), keyType);
                NullableTuple val = new NullableTuple((Tuple)result.get(3));


                NullablePartitionWritable wrappedKey = new NullablePartitionWritable(key);
                wrappedKey.setIndex(index);
                wrappedKey.setPartition((Integer)result.get(1));
                val.setIndex(index);


                try {
                    writer.write(wrappedKey, val);
                } catch (IOException ioe) {

View Full Code Here

            if (shuffleInput) {
                while (shuffleReader.next()) {
                    Object curKey = shuffleReader.getCurrentKey();
                    Iterable<Object> vals = shuffleReader.getCurrentValues();
                    if (isSkewedJoin) {
                        NullablePartitionWritable wrappedKey = new NullablePartitionWritable(
                                (PigNullableWritable) curKey);
                        wrappedKey.setPartition(-1);
                        curKey = wrappedKey;
                    }
                    for (Object val : vals) {
                        writer.write(curKey, val);
                    }
                }
            } else {
                while (reader.next()) {
                    if (isSkewedJoin) {
                        NullablePartitionWritable wrappedKey = new NullablePartitionWritable(
                                (PigNullableWritable) reader.getCurrentKey());
                        // Skewed join wraps key with NullablePartitionWritable
                        // The partitionIndex in NullablePartitionWritable is not serialized.
                        // So setting it here instead of the previous vertex POLocalRearrangeTez.
                        // Serializing it would add overhead for MR as well.
                        wrappedKey.setPartition(-1);
                        writer.write(wrappedKey, reader.getCurrentValue());
                    } else {
                        writer.write(reader.getCurrentKey(),
                                reader.getCurrentValue());
                    }

View Full Code Here

        pop.setInner(inner);
        PigNullableWritable k = HDataType.getWritableComparableTypes(key, keyType);
        pop.attachInput(k, db.iterator());
        if (keyType != DataType.BAG) {
            // test serialization
            NullablePartitionWritable wr;
            if (keyType == DataType.TUPLE) {
                BinSedesTuple tup = (BinSedesTuple) binfactory.newTupleNoCopy(((Tuple) k.getValueAsPigType()).getAll());
                wr = new NullablePartitionWritable(new NullableTuple(tup));
            } else {
                wr = new NullablePartitionWritable(k);
            }
            ByteArrayOutputStream baos = new ByteArrayOutputStream();
            DataOutputStream out = new DataOutputStream(baos);
            wr.write(out);
            byte[] arr = baos.toByteArray();
            ByteArrayInputStream bais = new ByteArrayInputStream(arr);
            DataInputStream in = new DataInputStream(bais);
            NullablePartitionWritable re = new NullablePartitionWritable();
            re.readFields(in);
            assertEquals(re, wr);
        }


        // we are not doing any optimization to remove
        // parts of the "value" which are present in the "key" in this

View Full Code Here

      }


            PigNullableWritable key =
                HDataType.getWritableComparableTypes(tuple.get(tupleKeyIdx), keyType);


            NullablePartitionWritable wrappedKey = new NullablePartitionWritable(key);


            NullableTuple val = new NullableTuple((Tuple)tuple.get(tupleValIdx));
            
            // Both the key and the value need the index.  The key needs it so
            // that it can be sorted on the index in addition to the key
            // value.  The value needs it so that POPackage can properly
            // assign the tuple to its slot in the projection.
            wrappedKey.setIndex(index);
            
            // set the partition
            wrappedKey.setPartition(partitionIndex);
            val.setIndex(index);
            oc.write(wrappedKey, val);
        }

View Full Code Here

      }


            PigNullableWritable key =
                HDataType.getWritableComparableTypes(tuple.get(tupleKeyIdx), keyType);


            NullablePartitionWritable wrappedKey = new NullablePartitionWritable(key);


            NullableTuple val = new NullableTuple((Tuple)tuple.get(tupleValIdx));
            
            // Both the key and the value need the index.  The key needs it so
            // that it can be sorted on the index in addition to the key
            // value.  The value needs it so that POPackage can properly
            // assign the tuple to its slot in the projection.
            wrappedKey.setIndex(index);
            
            // set the partition
            wrappedKey.setPartition(partitionIndex);
            val.setIndex(index);
            oc.write(wrappedKey, val);
        }

View Full Code Here

      }


            PigNullableWritable key =
                HDataType.getWritableComparableTypes(tuple.get(tupleKeyIdx), keyType);


            NullablePartitionWritable wrappedKey = new NullablePartitionWritable(key);


            NullableTuple val = new NullableTuple((Tuple)tuple.get(tupleValIdx));
            
            // Both the key and the value need the index.  The key needs it so
            // that it can be sorted on the index in addition to the key
            // value.  The value needs it so that POPackage can properly
            // assign the tuple to its slot in the projection.
            wrappedKey.setIndex(index);
            
            // set the partition
            wrappedKey.setPartition(partitionIndex);
            val.setIndex(index);
            oc.write(wrappedKey, val);
        }

View Full Code Here

      }


            PigNullableWritable key =
                HDataType.getWritableComparableTypes(tuple.get(tupleKeyIdx), keyType);


            NullablePartitionWritable wrappedKey = new NullablePartitionWritable(key);


            NullableTuple val = new NullableTuple((Tuple)tuple.get(tupleValIdx));
            
            // Both the key and the value need the index.  The key needs it so
            // that it can be sorted on the index in addition to the key
            // value.  The value needs it so that POPackage can properly
            // assign the tuple to its slot in the projection.
            wrappedKey.setIndex(index);
            
            // set the partition
            wrappedKey.setPartition(partitionIndex);
            val.setIndex(index);
            oc.write(wrappedKey, val);
        }

View Full Code Here

0 1

TOP

Related Classes of org.apache.pig.impl.io.NullablePartitionWritable

org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigGenericMapReduce$MapWithPartitionIndex

org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigMapReduce$MapWithPartitionIndex

org.apache.pig.backend.hadoop.executionengine.tez.plan.operator.POIdentityInOutTez

org.apache.pig.backend.hadoop.executionengine.tez.plan.operator.POPartitionRearrangeTez

org.apache.pig.test.TestPackage

java.io.IOException

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.