Examples of BytesWritable

org.apache.hadoop.io.BytesWritable
A byte sequence that is usable as a key or value. It is resizable and distinguishes between the size of the seqeunce and the current capacity. The hash function is the front of the md5 of the buffer. The sort order is the same as memcmp. @author Doug Cutting

Examples of org.apache.hadoop.io.BytesWritable


    byte[] buffer = new byte[size];
    for (int i = 0; i < buffer.length; i++) {
      buffer[i] = (byte)'A';
    }
    BytesWritable bytes = new BytesWritable(buffer);
    List<WALEntry<BytesWritable>> batch = Lists.newArrayList();
    long seqid = 0;
    long numBytes = 0;
    long count = 0;
    long start = System.currentTimeMillis();

View Full Code Here

Examples of org.apache.hadoop.io.BytesWritable

        for (int i = 0; i < splits.length; i++) {
          String mapId = jobId + "_map_" + idFormat.format(i);
          mapIds.add(mapId);
          buffer.reset();
          splits[i].write(buffer);
          BytesWritable split = new BytesWritable();
          split.set(buffer.getData(), 0, buffer.getLength());
          MapTask map = new MapTask(jobId, file.toString(), "tip_m_" + mapId, 
                                    mapId, i,
                                    splits[i].getClass().getName(),
                                    split);
          JobConf localConf = new JobConf(job);

View Full Code Here

Examples of org.apache.hadoop.io.BytesWritable

            Text t = (Text) from;
            to.bytes(t.getBytes(), t.getLength());
            return;
        }
        if (from instanceof BytesWritable) {
            BytesWritable b = (BytesWritable) from;
            to.bytes(b.getBytes(), b.getLength());
            return;
        }


        super.convert(from, to);
    }

View Full Code Here

Examples of org.apache.hadoop.io.BytesWritable

    assertEquals(md5HashKey1, md5HashKey2);
  }


  @Test
  public void testMD5HashForBytesWritableKey() throws IOException {
    BytesWritable key = new BytesWritable("abc123".getBytes());
    MD5Hash md5HashKey1 = HashUtility.getMD5Hash(key);
    MD5Hash md5HashKey2 = HashUtility.getMD5Hash(key);
    assertEquals(md5HashKey1, md5HashKey2);
  }

View Full Code Here

Examples of org.apache.hadoop.io.BytesWritable

  }


  @Test
  public void testReducerForBytesWritableKeyAndValue() throws IOException,
      InterruptedException {
    BytesWritable key = new BytesWritable("abc123".getBytes());
    HihoTuple hihoTuple = new HihoTuple();
    hihoTuple.setKey(key);


    BytesWritable value1 = new BytesWritable("value1".getBytes());
    ArrayList<BytesWritable> values = new ArrayList<BytesWritable>();
    values.add(value1);


    Reducer.Context context = mock(Reducer.Context.class);
    Counters counters = new Counters();

View Full Code Here

Examples of org.apache.hadoop.io.BytesWritable

  }


  @Test
  public void testReducerForBytesWritableKeyAndValue() throws IOException,
      InterruptedException {
    BytesWritable key = new BytesWritable("abc123".getBytes());
    HihoTuple hihoTuple = new HihoTuple();
    hihoTuple.setKey(key);


    BytesWritable value1 = new BytesWritable("value1".getBytes());
    ArrayList<BytesWritable> values = new ArrayList<BytesWritable>();
    values.add(value1);


    Reducer.Context context = mock(Reducer.Context.class);
    Counters counters = new Counters();

View Full Code Here

Examples of org.apache.hadoop.io.BytesWritable

  }


  @Test
  public void testReducerForBytesWritableKeyAndValue() throws IOException,
      InterruptedException {
    BytesWritable key = new BytesWritable("abc123".getBytes());
    HihoTuple hihoTuple = new HihoTuple();
    hihoTuple.setKey(key);


    HihoValue hihoValue1 = new HihoValue();
    HihoValue hihoValue2 = new HihoValue();
    BytesWritable value1 = new BytesWritable("value1".getBytes());
    BytesWritable value2 = new BytesWritable("value2".getBytes());
    hihoValue1.setVal(value1);
    hihoValue2.setVal(value2);
    hihoValue1.setIsOld(true);
    hihoValue2.setIsOld(false);
    ArrayList<HihoValue> values = new ArrayList<HihoValue>();

View Full Code Here

Examples of org.apache.hadoop.io.BytesWritable

  }


  @Test
  public void testReducerForBytesWritableKeyAndValue() throws IOException,
      InterruptedException {
    BytesWritable key = new BytesWritable("abc123".getBytes());
    HihoTuple hihoTuple = new HihoTuple();
    hihoTuple.setKey(key);


    HihoValue hihoValue1 = new HihoValue();
    HihoValue hihoValue2 = new HihoValue();
    BytesWritable value1 = new BytesWritable("value1".getBytes());
    BytesWritable value2 = new BytesWritable("value2".getBytes());
    hihoValue1.setVal(value1);
    hihoValue2.setVal(value2);
    hihoValue1.setIsOld(true);
    hihoValue2.setIsOld(false);
    ArrayList<HihoValue> values = new ArrayList<HihoValue>();

View Full Code Here

Examples of org.apache.hadoop.io.BytesWritable

    for(String outputFile : getAllFiles(dir)) {
      String name = (new File(outputFile)).getName();
      if(name.startsWith(prefix)) {
        SequenceFile.Reader reader = new SequenceFile.Reader(fs, new Path(outputFile), conf);
        LongWritable key = new LongWritable();
        BytesWritable value = new BytesWritable();
        while(reader.next(key, value)) {
          String body = new String(value.getBytes(), 0, value.getLength());
          if (bodies.contains(body)) {
            LOG.debug("Found event body: {}", body);
            bodies.remove(body);
            found++;
          }

View Full Code Here

Examples of org.apache.hadoop.io.BytesWritable

        System.arraycopy(content, eol + 1, raw, 0, raw.length);
        
        // populate key and values with the header and raw content.
        Text keyText = (Text)key;
        keyText.set(header);
        BytesWritable valueBytes = (BytesWritable)value;
        valueBytes.set(raw, 0, raw.length);


        // TODO: It would be best to start at the end of the gzip read but 
        // the bytes read in gzip don't match raw bytes in the file so we 
        // overshoot the next header.  With this current method you get
        // some false positives but don't miss records.

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.