Package org.apache.pig.impl.io

Examples of org.apache.pig.impl.io.PigNullableWritable


        @Override
        public void collect(Context oc, Tuple tuple)
                throws InterruptedException, IOException {

            Byte index = (Byte)tuple.get(0);
            PigNullableWritable key =
                HDataType.getWritableComparableTypes(tuple.get(1), keyType);
            NullableTuple val = new NullableTuple((Tuple)tuple.get(2));

            // Both the key and the value need the index.  The key needs it so
            // that it can be sorted on the index in addition to the key
            // value.  The value needs it so that POPackage can properly
            // assign the tuple to its slot in the projection.
            key.setIndex(index);
            val.setIndex(index);

            oc.write(key, val);
        }
View Full Code Here


                keyTuple = tuple.get(1);
            }


            Byte index = (Byte)tuple.get(0);
            PigNullableWritable key =
                HDataType.getWritableComparableTypes(keyTuple, DataType.TUPLE);
            NullableTuple val = new NullableTuple((Tuple)tuple.get(2));

            // Both the key and the value need the index.  The key needs it so
            // that it can be sorted on the index in addition to the key
            // value.  The value needs it so that POPackage can properly
            // assign the tuple to its slot in the projection.
            key.setIndex(index);
            val.setIndex(index);

            oc.write(key, val);
        }
View Full Code Here

        tupleValIdx--;
      } else {
        partitionIndex = (Integer)tuple.get(1);
      }

            PigNullableWritable key =
                HDataType.getWritableComparableTypes(tuple.get(tupleKeyIdx), keyType);

            NullablePartitionWritable wrappedKey = new NullablePartitionWritable(key);

            NullableTuple val = new NullableTuple((Tuple)tuple.get(tupleValIdx));
View Full Code Here

    private PhysicalOperator attachInputWithIndex(Tuple res) throws ExecException {
       
        // unwrap the first field of the tuple to get the wrapped value which
        // is expected by the inner plans, as well as the index of the associated
        // inner plan.
        PigNullableWritable fld = (PigNullableWritable)res.get(0);       
        // choose an inner plan to run based on the index set by
        // the POLocalRearrange operator and passed to this operator
        // by POMultiQueryPackage
        int index = fld.getIndex();
        index &= idxPart;                     

        curPlan = myPlans.get(index);
        if (!(curPlan.getRoots().get(0) instanceof PODemux)) {                            
            res.set(0, fld.getValueAsPigType());
        }
       
        curPlan.attachInput(res);
        return curPlan.getLeaves().get(0);
    }
View Full Code Here

     * restore it.
     */
    @Override
    public Tuple getValueTuple(PigNullableWritable keyWritable,
            NullableTuple ntup, int index) throws ExecException {
        PigNullableWritable origKey = this.keyWritable;
        this.keyWritable = keyWritable;
        Tuple retTuple = super.getValueTuple(keyWritable, ntup, index);
        this.keyWritable = origKey;
        return retTuple;
    }
View Full Code Here

        Packager pkgr = packagers.get(index);

        // check to see if we need to unwrap the key. The keys may be
        // wrapped inside a tuple by LocalRearrange operator when jobs
        // with different map key types are merged
        PigNullableWritable curKey = keyWritable;
        if (!sameMapKeyType && !inCombiner && isKeyWrapped.get(index)) {
            Tuple tup = (Tuple) keyWritable.getValueAsPigType();
            curKey = HDataType.getWritableComparableTypes(tup.get(0),
                    pkgr.getKeyType());
            curKey.setIndex(origIndex);
        }

        pkgr.attachInput(curKey, bags, readOnce);

        Result res = pkgr.getNext();
        pkgr.detachInput();
        detachInput();

        Tuple tuple = (Tuple)res.result;

        // the object present in the first field
        // of the tuple above is the real data without
        // index information - this is because the
        // package above, extracts the real data out of
        // the PigNullableWritable object - we are going to
        // give this result tuple to a PODemux operator
        // which needs a PigNullableWritable first field so
        // it can figure out the index. Therefore we need
        // to add index to the first field of the tuple.

        Object obj = tuple.get(0);
        if (obj instanceof PigNullableWritable) {
            ((PigNullableWritable)obj).setIndex(origIndex);
        }
        else {
            PigNullableWritable myObj = null;
            if (obj == null) {
                myObj = new NullableUnknownWritable();
                myObj.setNull(true);
            }
            else {
                myObj = HDataType.getWritableComparableTypes(obj, HDataType.findTypeFromNullableWritable(curKey));
            }
            myObj.setIndex(origIndex);
            tuple.set(0, myObj);
        }
        // illustrator markup has been handled by "pkgr"
        return res;
    }
View Full Code Here

        // should never come here
        return null;
    }

    public static PigNullableWritable getWritableComparableTypes(byte type) throws ExecException{
        PigNullableWritable wcKey = null;
         switch (type) {
        case DataType.BAG:
            wcKey = defDB;
            break;
        case DataType.BOOLEAN:
View Full Code Here

        @SuppressWarnings("unchecked")
        @Override
        public int compare(WritableComparable a, WritableComparable b)
        {
            PigNullableWritable wa = (PigNullableWritable)a;
            PigNullableWritable wb = (PigNullableWritable)b;
            if ((wa.getIndex() & PigNullableWritable.mqFlag) != 0) { // this is a multi-query index
                if ((wa.getIndex() & PigNullableWritable.idxSpace) < (wb.getIndex() & PigNullableWritable.idxSpace)) return -1;
                else if ((wa.getIndex() & PigNullableWritable.idxSpace) > (wb.getIndex() & PigNullableWritable.idxSpace)) return 1;
                // If equal, we fall through
            }

            // wa and wb are guaranteed to be not null, POLocalRearrange will create a tuple anyway even if main key and secondary key
            // are both null; however, main key can be null, we need to check for that using the same logic we have in PigNullableWritable
            Object valuea = null;
            Object valueb = null;
            try {
                // Get the main key from compound key
                valuea = ((Tuple)wa.getValueAsPigType()).get(0);
                valueb = ((Tuple)wb.getValueAsPigType()).get(0);
            } catch (ExecException e) {
                throw new RuntimeException("Unable to access tuple field", e);
            }
            if (!wa.isNull() && !wb.isNull()) {

                int result = DataType.compare(valuea, valueb);

                // If any of the field inside tuple is null, then we do not merge keys
                // See PIG-927
                if (result == 0 && valuea instanceof Tuple && valueb instanceof Tuple)
                {
                    try {
                        for (int i=0;i<((Tuple)valuea).size();i++)
                            if (((Tuple)valueb).get(i)==null)
                                return (wa.getIndex()&PigNullableWritable.idxSpace) - (wb.getIndex()&PigNullableWritable.idxSpace);
                    } catch (ExecException e) {
                        throw new RuntimeException("Unable to access tuple field", e);
                    }
                }
                return result;
            } else if (valuea==null && valueb==null) {
                // If they're both null, compare the indicies
                if ((wa.getIndex() & PigNullableWritable.idxSpace) < (wb.getIndex() & PigNullableWritable.idxSpace)) return -1;
                else if ((wa.getIndex() & PigNullableWritable.idxSpace) > (wb.getIndex() & PigNullableWritable.idxSpace)) return 1;
                else return 0;
            }
            else if (valuea==null) return -1;
            else return 1;
        }
View Full Code Here

        }
        // ITIterator iti = new TestPackage.ITIterator(db.iterator());
        POPackage pop = new POPackage(new OperatorKey("", r.nextLong()));
        pop.setNumInps(2);
        pop.getPkgr().setInner(inner);
        PigNullableWritable k = HDataType.getWritableComparableTypes(key, keyType);
        pop.attachInput(k, db.iterator());
        if (keyType != DataType.BAG) {
            // test serialization
            NullablePartitionWritable wr;
            if (keyType == DataType.TUPLE) {
                BinSedesTuple tup = (BinSedesTuple) binfactory.newTupleNoCopy(((Tuple) k.getValueAsPigType()).getAll());
                wr = new NullablePartitionWritable(new NullableTuple(tup));
            } else {
                wr = new NullablePartitionWritable(k);
            }
            ByteArrayOutputStream baos = new ByteArrayOutputStream();
View Full Code Here

    @Override
    public int getPartition(PigNullableWritable key, Writable value,
            int numPartitions) {
        try {
            Tuple t = (Tuple)key.getValueAsPigType();
            PigNullableWritable realKey = HDataType.getWritableComparableTypes(t.get(0), kt);
            return super.getPartition(realKey, value, numPartitions);
        } catch (ExecException e) {
            throw new RuntimeException(e);
        }
    }
View Full Code Here

TOP

Related Classes of org.apache.pig.impl.io.PigNullableWritable

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.