Examples of BytesWritable

org.apache.hadoop.io.BytesWritable
A byte sequence that is usable as a key or value. It is resizable and distinguishes between the size of the seqeunce and the current capacity. The hash function is the front of the md5 of the buffer. The sort order is the same as memcmp. @author Doug Cutting

Examples of org.apache.hadoop.io.BytesWritable

        byte[] keyBytes = null, valueBytes;


        keyBytes = new byte[keyBuffer.remaining()];
        keyBuffer.get(keyBytes);


        BytesWritable key = new BytesWritable(keyBytes);


        ArrayList<BytesWritable> valueList = new ArrayList();


        while(iterator.hasNext()) {
            ByteBuffer writable = iterator.next().datum();
            writable.rewind();
            // BytesWritable writable = iterator.next();
            valueBytes = null;
            valueBytes = new byte[writable.remaining()];
            writable.get(valueBytes);


            BytesWritable value = new BytesWritable(valueBytes);
            valueList.add(value);


        }


        writer.write(key, valueList.iterator(), reporter);

View Full Code Here

Examples of org.apache.hadoop.io.BytesWritable


        public void map(LongWritable lineNumber,
                        Text line,
                        OutputCollector<BytesWritable, BytesWritable> collector,
                        Reporter reporter) throws IOException {
            collector.collect(new BytesWritable(line.getBytes()), value);
        }

View Full Code Here

Examples of org.apache.hadoop.io.BytesWritable

        public void configure(JobConf job) {
            StringBuilder builder = new StringBuilder();
            int size = job.getInt("value.size", -1);
            for(int i = 0; i < size; i++)
                builder.append('a');
            this.value = new BytesWritable(builder.toString().getBytes());
        }

View Full Code Here

Examples of org.apache.hadoop.io.BytesWritable

        short numTuples = 0;
        ByteArrayOutputStream stream = new ByteArrayOutputStream();
        DataOutputStream valueStream = new DataOutputStream(stream);


        while(iterator.hasNext()) {
            BytesWritable writable = iterator.next();
            byte[] valueBytes = writable.get();
            int offsetTillNow = 0;


            // Read node Id
            if(this.nodeId == -1)
                this.nodeId = ByteUtils.readInt(valueBytes, offsetTillNow);
            offsetTillNow += ByteUtils.SIZE_OF_INT;


            // Read partition id
            if(this.partitionId == -1)
                this.partitionId = ByteUtils.readInt(valueBytes, offsetTillNow);
            offsetTillNow += ByteUtils.SIZE_OF_INT;


            // Read chunk id
            if(this.chunkId == -1)
                this.chunkId = ReadOnlyUtils.chunk(key.get(), getNumChunks());


            // Read replica type
            if(getSaveKeys()) {
                if(this.replicaType == -1)
                    this.replicaType = (int) ByteUtils.readBytes(valueBytes,
                                                                 offsetTillNow,
                                                                 ByteUtils.SIZE_OF_BYTE);
                offsetTillNow += ByteUtils.SIZE_OF_BYTE;
            }


            int valueLength = writable.getSize() - offsetTillNow;
            if(getSaveKeys()) {
                // Write ( key_length, value_length, key,
                // value )
                valueStream.write(valueBytes, offsetTillNow, valueLength);
            } else {

View Full Code Here

Examples of org.apache.hadoop.io.BytesWritable

        short numTuples = 0;
        ByteArrayOutputStream stream = new ByteArrayOutputStream();
        DataOutputStream valueStream = new DataOutputStream(stream);


        while(iterator.hasNext()) {
            BytesWritable writable = iterator.next();
            byte[] valueBytes = writable.get();
            int offsetTillNow = 0;


            // Read node Id
            if(this.nodeId == -1)
                this.nodeId = ByteUtils.readInt(valueBytes, offsetTillNow);
            offsetTillNow += ByteUtils.SIZE_OF_INT;


            // Read partition id
            if(this.partitionId == -1)
                this.partitionId = ByteUtils.readInt(valueBytes, offsetTillNow);
            offsetTillNow += ByteUtils.SIZE_OF_INT;


            // Read replica type
            if(getSaveKeys()) {
                if(this.replicaType == -1)
                    this.replicaType = (int) ByteUtils.readBytes(valueBytes,
                                                                 offsetTillNow,
                                                                 ByteUtils.SIZE_OF_BYTE);
                offsetTillNow += ByteUtils.SIZE_OF_BYTE;
            }


            int valueLength = writable.getSize() - offsetTillNow;
            if(getSaveKeys()) {
                // Write ( key_length, value_length, key,
                // value )
                valueStream.write(valueBytes, offsetTillNow, valueLength);
            } else {

View Full Code Here

Examples of org.apache.hadoop.io.BytesWritable


    @Override
    void write(Object obj) throws IOException {
      long rawDataSize = 0;
      if (obj != null) {
        BytesWritable val =
            ((BinaryObjectInspector) inspector).getPrimitiveWritableObject(obj);
        stream.write(val.getBytes(), 0, val.getLength());
        length.write(val.getLength());


        // Raw data size is the length of the BytesWritable, i.e. the number of bytes
        rawDataSize = val.getLength();
      }
      super.write(obj, rawDataSize);
    }

View Full Code Here

Examples of org.apache.hadoop.io.BytesWritable


      @Override
      public Object next(Object previous) throws IOException {
        if (nextCalls == 0) {
          nextCalls++;
          return new BytesWritable("a".getBytes());
        }


        throw new IOException("next should only be called once");
      }

View Full Code Here

Examples of org.apache.hadoop.io.BytesWritable

    }
    return result;
  }


  public static BytesWritable bytes(int... items) {
    BytesWritable result = new BytesWritable();
    result.setSize(items.length);
    for(int i=0; i < items.length; ++i) {
      result.getBytes()[i] = (byte) items[i];
    }
    return result;
  }

View Full Code Here

Examples of org.apache.hadoop.io.BytesWritable

        byte[] keyBytes = null, valueBytes;


        keyBytes = new byte[keyBuffer.remaining()];
        keyBuffer.get(keyBytes);


        BytesWritable key = new BytesWritable(keyBytes);


        ArrayList<BytesWritable> valueList = new ArrayList();


        while(iterator.hasNext()) {
            ByteBuffer writable = iterator.next().datum();
            writable.rewind();
            // BytesWritable writable = iterator.next();
            valueBytes = null;
            valueBytes = new byte[writable.remaining()];
            writable.get(valueBytes);


            BytesWritable value = new BytesWritable(valueBytes);
            valueList.add(value);


        }


        writer.write(key, valueList.iterator(), reporter);

View Full Code Here

Examples of org.apache.hadoop.io.BytesWritable

        this.keySerializer = Utils.nonNull(keySerializer);
        this.valueSerializer = Utils.nonNull(valueSerializer);
    }


    public void collect(K key, V value) throws IOException {
        innerCollector.collect(new BytesWritable(keySerializer.toBytes(key)),
                               new BytesWritable(valueSerializer.toBytes(value)));
    }

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.