Examples of PigNullableWritable

org.apache.pig.impl.io.PigNullableWritable
A base class for all types that pig uses to move data between map and reduce. It implements WritableComparable so that compareTo etc. can be called. It also wraps a WritableComparable 'value'. This is set by each different type to be an object of its specific type. It also provides a getIndex() and setIndex() calls that are used to get and set the index. These can be used by LocalRearrange, the partitioner, and Package to determine the index. Index and the null indicator are packed into one byte to save space.

Examples of org.apache.pig.impl.io.PigNullableWritable

    private PhysicalOperator attachInputWithIndex(Tuple res) throws ExecException {
        
        // unwrap the first field of the tuple to get the wrapped value which
        // is expected by the inner plans, as well as the index of the associated
        // inner plan.
        PigNullableWritable fld = (PigNullableWritable)res.get(0);        
        // choose an inner plan to run based on the index set by
        // the POLocalRearrange operator and passed to this operator
        // by POMultiQueryPackage
        int index = fld.getIndex();
        index &= idxPart;                      


        curPlan = myPlans.get(index);
        if (!(curPlan.getRoots().get(0) instanceof PODemux)) {                             
            res.set(0, fld.getValueAsPigType());
        }
        
        curPlan.attachInput(res);
        return curPlan.getLeaves().get(0);
    }

View Full Code Here

Examples of org.apache.pig.impl.io.PigNullableWritable

        POPackage pack = packages.get(index);
        
        // check to see if we need to unwrap the key. The keys may be
        // wrapped inside a tuple by LocalRearrange operator when jobs  
        // with different map key types are merged
        PigNullableWritable curKey = myKey;
        if (!sameMapKeyType && !inCombiner && isKeyWrapped.get(index)) {                                       
            Tuple tup = (Tuple)myKey.getValueAsPigType();
            curKey = HDataType.getWritableComparableTypes(tup.get(0), pack.getKeyType());
            curKey.setIndex(origIndex);
        }
            
        pack.attachInput(curKey, tupIter);


        Result res = pack.getNext(t);
        pack.detachInput();
        
        Tuple tuple = (Tuple)res.result;


        // the object present in the first field
        // of the tuple above is the real data without
        // index information - this is because the
        // package above, extracts the real data out of
        // the PigNullableWritable object - we are going to
        // give this result tuple to a PODemux operator
        // which needs a PigNullableWritable first field so
        // it can figure out the index. Therefore we need
        // to add index to the first field of the tuple.
                
        Object obj = tuple.get(0);
        if (obj instanceof PigNullableWritable) {
            ((PigNullableWritable)obj).setIndex(origIndex);
        }
        else {
            PigNullableWritable myObj = null;
            if (obj == null) {
                myObj = new NullableUnknownWritable();
                myObj.setNull(true);
            }
            else {
                myObj = HDataType.getWritableComparableTypes(obj, (byte)0);
            }
            myObj.setIndex(origIndex);
            tuple.set(0, myObj);
        }
        
        return res;
    }

View Full Code Here

Examples of org.apache.pig.impl.io.PigNullableWritable

        // should never come here
        return null;
    }
    
    public static PigNullableWritable getWritableComparableTypes(byte type) throws ExecException{
        PigNullableWritable wcKey = null;
         switch (type) {
        case DataType.BAG:
            wcKey = defDB;
            break;
        case DataType.BOOLEAN:

View Full Code Here

Examples of org.apache.pig.impl.io.PigNullableWritable

        Pair <Integer, Integer> indexes;
        Integer curIndex = -1;
        Tuple keyTuple = TupleFactory.getInstance().newTuple(1);


        // extract the key from nullablepartitionwritable
        PigNullableWritable key = ((NullablePartitionWritable) wrappedKey).getKey();


        try {
            keyTuple.set(0, key.getValueAsPigType());
        } catch (ExecException e) {
            return -1;
        }


        // if the key is not null and key 
        if (key instanceof NullableTuple && key.getValueAsPigType() != null) {
            keyTuple = (Tuple)key.getValueAsPigType();
        }


        // if the partition file is empty, use numPartitions
        totalReducers = (totalReducers > 0) ? totalReducers : numPartitions;

View Full Code Here

Examples of org.apache.pig.impl.io.PigNullableWritable


        @SuppressWarnings("unchecked")
    @Override
        public int compare(WritableComparable a, WritableComparable b)
        {
            PigNullableWritable wa = (PigNullableWritable)a;
            PigNullableWritable wb = (PigNullableWritable)b;
            if ((wa.getIndex() & PigNullableWritable.mqFlag) != 0) { // this is a multi-query index
                if ((wa.getIndex() & PigNullableWritable.idxSpace) < (wb.getIndex() & PigNullableWritable.idxSpace)) return -1;
                else if ((wa.getIndex() & PigNullableWritable.idxSpace) > (wb.getIndex() & PigNullableWritable.idxSpace)) return 1;
                // If equal, we fall through
            }
            
            // wa and wb are guaranteed to be not null, POLocalRearrange will create a tuple anyway even if main key and secondary key
            // are both null; however, main key can be null, we need to check for that using the same logic we have in PigNullableWritable
            Object valuea = null;
            Object valueb = null;
            try {
                // Get the main key from compound key
                valuea = ((Tuple)wa.getValueAsPigType()).get(0);
                valueb = ((Tuple)wb.getValueAsPigType()).get(0);
            } catch (ExecException e) {
                throw new RuntimeException("Unable to access tuple field", e);
            }
            if (!wa.isNull() && !wb.isNull()) {
                
                int result = DataType.compare(valuea, valueb);
                
                // If any of the field inside tuple is null, then we do not merge keys
                // See PIG-927
                if (result == 0 && valuea instanceof Tuple && valueb instanceof Tuple)
                {
                    try {
                        for (int i=0;i<((Tuple)valuea).size();i++)
                            if (((Tuple)valueb).get(i)==null)
                                return (wa.getIndex()&PigNullableWritable.idxSpace) - (wb.getIndex()&PigNullableWritable.idxSpace);
                    } catch (ExecException e) {
                        throw new RuntimeException("Unable to access tuple field", e);
                    }
                }
                return result;
            } else if (valuea==null && valueb==null) {
                // If they're both null, compare the indicies
                if ((wa.getIndex() & PigNullableWritable.idxSpace) < (wb.getIndex() & PigNullableWritable.idxSpace)) return -1;
                else if ((wa.getIndex() & PigNullableWritable.idxSpace) > (wb.getIndex() & PigNullableWritable.idxSpace)) return 1;
                else return 0;
            }
            else if (valuea==null) return -1; 
            else return 1;
        }

View Full Code Here

Examples of org.apache.pig.impl.io.PigNullableWritable

        @Override
        public void collect(Context oc, Tuple tuple) 
                throws InterruptedException, IOException {
            
            Byte index = (Byte)tuple.get(0);
            PigNullableWritable key =
                HDataType.getWritableComparableTypes(tuple.get(1), keyType);
            NullableTuple val = new NullableTuple((Tuple)tuple.get(2));
            
            // Both the key and the value need the index.  The key needs it so
            // that it can be sorted on the index in addition to the key
            // value.  The value needs it so that POPackage can properly
            // assign the tuple to its slot in the projection.
            key.setIndex(index);
            val.setIndex(index);           
              
            oc.write(key, val);
        }

View Full Code Here

Examples of org.apache.pig.impl.io.PigNullableWritable

                keyTuple = tuple.get(1);
            }
            


            Byte index = (Byte)tuple.get(0);
            PigNullableWritable key =
                HDataType.getWritableComparableTypes(keyTuple, DataType.TUPLE);
            NullableTuple val = new NullableTuple((Tuple)tuple.get(2));
            
            // Both the key and the value need the index.  The key needs it so
            // that it can be sorted on the index in addition to the key
            // value.  The value needs it so that POPackage can properly
            // assign the tuple to its slot in the projection.
            key.setIndex(index);
            val.setIndex(index);


            oc.write(key, val);
        }

View Full Code Here

Examples of org.apache.pig.impl.io.PigNullableWritable

        tupleValIdx--;
      } else {
        partitionIndex = (Integer)tuple.get(1);
      }


            PigNullableWritable key =
                HDataType.getWritableComparableTypes(tuple.get(tupleKeyIdx), DataType.TUPLE);


            NullablePartitionWritable wrappedKey = new NullablePartitionWritable(key);


            NullableTuple val = new NullableTuple((Tuple)tuple.get(tupleValIdx));

View Full Code Here

Examples of org.apache.pig.impl.io.PigNullableWritable

                        Result redRes = leaf.getNext(DUMMYTUPLE);
                        
                        if(redRes.returnStatus==POStatus.STATUS_OK){
                            Tuple tuple = (Tuple)redRes.result;
                            Byte index = (Byte)tuple.get(0);
                            PigNullableWritable outKey =
                                HDataType.getWritableComparableTypes(tuple.get(1), this.keyType);
                            NullableTuple val =
                                new NullableTuple((Tuple)tuple.get(2));
                            // Both the key and the value need the index.  The key needs it so
                            // that it can be sorted on the index in addition to the key
                            // value.  The value needs it so that POPackage can properly
                            // assign the tuple to its slot in the projection.
                            outKey.setIndex(index);
                            val.setIndex(index);


                            oc.write(outKey, val);


                            continue;

View Full Code Here

Examples of org.apache.pig.impl.io.PigNullableWritable

        }
        //ITIterator iti = new TestPackage.ITIterator(db.iterator());
        POPackage pop = new POPackage(new OperatorKey("", r.nextLong()));
        pop.setNumInps(2);
        pop.setInner(inner);
        PigNullableWritable k = HDataType.getWritableComparableTypes(key, (byte)0);
        pop.attachInput(k, db.iterator());
        
        // we are not doing any optimization to remove
        // parts of the "value" which are present in the "key" in this
        // unit test - so set up the "keyInfo" accordingly in

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.