Examples of IntWritable

org.apache.hadoop.io.IntWritable
A WritableComparable for ints.

Examples of org.apache.hadoop.io.IntWritable

        15 * 1024, CompressionKind.NONE, 100, 10000);
    OrcStruct row = new OrcStruct(types.get(0).getFieldNamesList());
    OrcUnion union = new OrcUnion();
    row.setFieldValue(1, union);
    row.setFieldValue(0, Timestamp.valueOf("2000-03-12 15:00:00"));
    union.set((byte) 0, new IntWritable(42));
    writer.addRow(row);
    row.setFieldValue(0, Timestamp.valueOf("2000-03-20 12:00:00.123456789"));
    union.set((byte)1, new Text("hello"));
    writer.addRow(row);
    row.setFieldValue(0, null);
    row.setFieldValue(1, null);
    writer.addRow(row);
    row.setFieldValue(1, union);
    union.set((byte) 0, null);
    writer.addRow(row);
    union.set((byte) 1, null);
    writer.addRow(row);
    union.set((byte) 0, new IntWritable(200000));
    row.setFieldValue(0, Timestamp.valueOf("1900-01-01 00:00:00"));
    writer.addRow(row);
    for(int i=1900; i < 2200; ++i) {
      row.setFieldValue(0, Timestamp.valueOf(i + "-05-05 12:34:56." + i));
      if ((i & 1) == 0) {
        union.set((byte) 0, new IntWritable(i*i));
      } else {
        union.set((byte) 1, new Text(new Integer(i*i).toString()));
      }
      writer.addRow(row);
    }
    // let's add a lot of constant rows to test the rle
    row.setFieldValue(0, null);
    union.set((byte) 0, new IntWritable(1732050807));
    for(int i=0; i < 5000; ++i) {
      writer.addRow(row);
    }
    union.set((byte) 0, new IntWritable(0));
    writer.addRow(row);
    union.set((byte) 0, new IntWritable(10));
    writer.addRow(row);
    union.set((byte) 0, new IntWritable(138));
    writer.addRow(row);
    writer.close();
    ReaderWriterProfiler.setProfilerOptions(conf);
    Reader reader = OrcFile.createReader(fs, testFilePath, conf);
    assertEquals(false, reader.getMetadataKeys().iterator().hasNext());
    assertEquals(5309, reader.getNumberOfRows());
    int stripeCount = 0;
    int rowCount = 0;
    long currentOffset = -1;
    for(StripeInformation stripe: reader.getStripes()) {
      stripeCount += 1;
      rowCount += stripe.getNumberOfRows();
      if (currentOffset < 0) {
        currentOffset = stripe.getOffset() + stripe.getIndexLength() +
            stripe.getDataLength() + stripe.getFooterLength();
      } else {
        assertEquals(currentOffset, stripe.getOffset());
        currentOffset += stripe.getIndexLength() +
            stripe.getDataLength() + stripe.getFooterLength();
      }
    }
    assertEquals(reader.getNumberOfRows(), rowCount);
    assertEquals(2, stripeCount);
    assertEquals(reader.getContentLength(), currentOffset);
    RecordReader rows = reader.rows(null);
    assertEquals(0, rows.getRowNumber());
    assertEquals(0.0, rows.getProgress(), 0.000001);
    assertEquals(true, rows.hasNext());
    OrcLazyStruct lazyRow = (OrcLazyStruct) rows.next(null);
    row = (OrcStruct) lazyRow.materialize();
    inspector = reader.getObjectInspector();
    assertEquals("struct<time:timestamp,union:uniontype<int,string>>",
        inspector.getTypeName());
    assertEquals(Timestamp.valueOf("2000-03-12 15:00:00"),
        ((TimestampWritable) ((OrcLazyTimestamp) row.getFieldValue(0)).materialize()).getTimestamp());
    union = (OrcUnion) ((OrcLazyUnion) row.getFieldValue(1)).materialize();
    assertEquals(0, union.getTag());
    assertEquals(new IntWritable(42), union.getObject());
    lazyRow = (OrcLazyStruct) rows.next(lazyRow);
    row = (OrcStruct) lazyRow.materialize();
    assertEquals(Timestamp.valueOf("2000-03-20 12:00:00.123456789"),
        ((TimestampWritable) ((OrcLazyTimestamp) row.getFieldValue(0)).materialize()).getTimestamp());
    ((OrcLazyUnion) row.getFieldValue(1)).materialize();
    assertEquals(1, union.getTag());
    assertEquals(new Text("hello"), union.getObject());
    lazyRow = (OrcLazyStruct) rows.next(lazyRow);
    row = (OrcStruct) lazyRow.materialize();
    assertEquals(null, ((OrcLazyObject) row.getFieldValue(0)).materialize());
    assertEquals(null, ((OrcLazyObject) row.getFieldValue(1)).materialize());
    lazyRow = (OrcLazyStruct) rows.next(lazyRow);
    row = (OrcStruct) lazyRow.materialize();
    assertEquals(null, ((OrcLazyObject) row.getFieldValue(0)).materialize());
    union = (OrcUnion) ((OrcLazyUnion) row.getFieldValue(1)).materialize();
    assertEquals(0, union.getTag());
    assertEquals(null, union.getObject());
    lazyRow = (OrcLazyStruct) rows.next(lazyRow);
    row = (OrcStruct) lazyRow.materialize();
    assertEquals(null, ((OrcLazyObject) row.getFieldValue(0)).materialize());
    ((OrcLazyUnion) row.getFieldValue(1)).materialize();
    assertEquals(1, union.getTag());
    assertEquals(null, union.getObject());
    lazyRow = (OrcLazyStruct) rows.next(lazyRow);
    row = (OrcStruct) lazyRow.materialize();
    assertEquals(Timestamp.valueOf("1900-01-01 00:00:00"),
        ((TimestampWritable) ((OrcLazyTimestamp) row.getFieldValue(0)).materialize()).getTimestamp());
    ((OrcLazyUnion) row.getFieldValue(1)).materialize();
    assertEquals(new IntWritable(200000), union.getObject());
    for(int i=1900; i < 2200; ++i) {
      lazyRow = (OrcLazyStruct) rows.next(lazyRow);
      row = (OrcStruct) lazyRow.materialize();
      assertEquals(Timestamp.valueOf(i + "-05-05 12:34:56." + i),
          ((TimestampWritable) ((OrcLazyTimestamp) row.getFieldValue(0)).materialize()).getTimestamp());
      ((OrcLazyUnion) row.getFieldValue(1)).materialize();
      if ((i & 1) == 0) {
        assertEquals(0, union.getTag());
        assertEquals(new IntWritable(i*i), union.getObject());
      } else {
        assertEquals(1, union.getTag());
        assertEquals(new Text(new Integer(i*i).toString()), union.getObject());
      }
    }
    for(int i=0; i < 5000; ++i) {
      lazyRow = (OrcLazyStruct) rows.next(lazyRow);
      row = (OrcStruct) lazyRow.materialize();
      ((OrcLazyUnion) row.getFieldValue(1)).materialize();
      assertEquals(new IntWritable(1732050807), union.getObject());
    }
    lazyRow = (OrcLazyStruct) rows.next(lazyRow);
    row = (OrcStruct) lazyRow.materialize();
    ((OrcLazyUnion) row.getFieldValue(1)).materialize();
    assertEquals(new IntWritable(0), union.getObject());
    lazyRow = (OrcLazyStruct) rows.next(lazyRow);
    row = (OrcStruct) lazyRow.materialize();
    ((OrcLazyUnion) row.getFieldValue(1)).materialize();
    assertEquals(new IntWritable(10), union.getObject());
    lazyRow = (OrcLazyStruct) rows.next(lazyRow);
    row = (OrcStruct) lazyRow.materialize();
    ((OrcLazyUnion) row.getFieldValue(1)).materialize();
    assertEquals(new IntWritable(138), union.getObject());
    assertEquals(false, rows.hasNext());
    assertEquals(1.0, rows.getProgress(), 0.00001);
    assertEquals(reader.getNumberOfRows(), rows.getRowNumber());
    rows.close();
  }

View Full Code Here

Examples of org.apache.hadoop.io.IntWritable

    } else {
      assertEquals(expected.short1.shortValue(), short1.get());
    }


    OrcLazyInt lazyInt1 = (OrcLazyInt) row.getFieldValue(3);
    IntWritable int1 = (IntWritable) lazyInt1.materialize();
    if (int1 == null) {
      assertNull(expected.int1);
    } else {
      assertEquals(expected.int1.intValue(), int1.get());
    }


    OrcLazyLong lazyLong1 = (OrcLazyLong) row.getFieldValue(4);
    LongWritable long1 = (LongWritable) lazyLong1.materialize();
    if (long1 == null) {
      assertNull(expected.long1);
    } else {
      assertEquals(expected.long1.longValue(), long1.get());
    }


    OrcLazyShort lazyShort2 = (OrcLazyShort) row.getFieldValue(5);
    ShortWritable short2 = (ShortWritable) lazyShort2.materialize();
    if (short2 == null) {
      assertNull(expected.short2);
    } else {
      assertEquals(expected.short2.shortValue(), short2.get());
    }


    OrcLazyInt lazyInt2 = (OrcLazyInt) row.getFieldValue(6);
    IntWritable int2 = (IntWritable) lazyInt2.materialize();
    if (int2 == null) {
      assertNull(expected.int2);
    } else {
      assertEquals(expected.int2.intValue(), int2.get());
    }


    OrcLazyLong lazyLong2 = (OrcLazyLong) row.getFieldValue(7);
    LongWritable long2 = (LongWritable) lazyLong2.materialize();
    if (long2 == null) {
      assertNull(expected.long2);
    } else {
      assertEquals(expected.long2.longValue(), long2.get());
    }


    OrcLazyShort lazyShort3 = (OrcLazyShort) row.getFieldValue(8);
    ShortWritable short3 = (ShortWritable) lazyShort3.materialize();
    if (short3 == null) {
      assertNull(expected.short3);
    } else {
      assertEquals(expected.short3.shortValue(), short3.get());
    }


    OrcLazyInt lazyInt3 = (OrcLazyInt) row.getFieldValue(9);
    IntWritable int3 = (IntWritable) lazyInt3.materialize();
    if (int3 == null) {
      assertNull(expected.int3);
    } else {
      assertEquals(expected.int3.intValue(), int3.get());
    }


    OrcLazyLong lazyLong3 = (OrcLazyLong) row.getFieldValue(10);
    LongWritable long3 = (LongWritable) lazyLong3.materialize();
    if (long3 == null) {

View Full Code Here

Examples of org.apache.hadoop.io.IntWritable

  private int latestValue() {
    return latestValue;
  }


  IntWritable createWritable(Object previous, int v) throws IOException {
    IntWritable result = null;
    if (previous == null) {
      result = new IntWritable();
    } else {
      result = (IntWritable) previous;
    }
    result.set(v);
    return result;
  }

View Full Code Here

Examples of org.apache.hadoop.io.IntWritable

  }




  @Override
  public Object next(Object previous) throws IOException {
    IntWritable result = null;
    if (valuePresent) {
      result = createWritable(previous, readInt());
    }
    return result;
  }

View Full Code Here

Examples of org.apache.hadoop.io.IntWritable

      int getCalls = 0;


      @Override
      public Object get(long currentRow, Object previous) throws IOException {
        if (getCalls == 0) {
          return new IntWritable(1);
        }


        throw new IOException("get should only be called once");
      }

View Full Code Here

Examples of org.apache.hadoop.io.IntWritable

        Reporter reporter) throws IOException {
      int sum = 0;
      while (values.hasNext()) {
        sum += ((IntWritable) values.next()).get();
      }
      output.collect(key, new IntWritable(sum));
    }

View Full Code Here

Examples of org.apache.hadoop.io.IntWritable

   * @return a Counter
   */
  public static Counters fromEscapedCompactString(String compactString) 
  throws ParseException {
    Counters counters = new Counters();
    IntWritable index = new IntWritable(0);
    
    // Get the group to work on
    String groupString = 
      getBlock(compactString, GROUP_OPEN, GROUP_CLOSE, index);
    
    while (groupString != null) {
      IntWritable groupIndex = new IntWritable(0);
      
      // Get the actual name
      String groupName = 
        getBlock(groupString, UNIT_OPEN, UNIT_CLOSE, groupIndex);
      groupName = unescape(groupName);
      
      // Get the display name
      String groupDisplayName = 
        getBlock(groupString, UNIT_OPEN, UNIT_CLOSE, groupIndex);
      groupDisplayName = unescape(groupDisplayName);
      
      // Get the counters
      Group group = counters.getGroup(groupName);
      group.setDisplayName(groupDisplayName);
      
      String counterString = 
        getBlock(groupString, COUNTER_OPEN, COUNTER_CLOSE, groupIndex);
      
      while (counterString != null) {
        IntWritable counterIndex = new IntWritable(0);
        
        // Get the actual name
        String counterName = 
          getBlock(counterString, UNIT_OPEN, UNIT_CLOSE, counterIndex);
        counterName = unescape(counterName);

View Full Code Here

Examples of org.apache.hadoop.io.IntWritable

            " to archive.");
        copyData(srcStatus.getPath(), input, partStream, reporter);
        towrite = relPath.toString() + " file " + partname + " " + startPos
        + " " + srcStatus.getLen() + " ";
      }
      out.collect(new IntWritable(hash), new Text(towrite));
    }

View Full Code Here

Examples of org.apache.hadoop.io.IntWritable

    private JobID jobId;
    private long lastFetchTime;
    private boolean fetchAgain;
     
    public FetchStatus(JobID jobId, int numMaps) {
      this.fromEventId = new IntWritable(0);
      this.jobId = jobId;
      this.allMapEvents = new ArrayList<TaskCompletionEvent>(numMaps);
    }

View Full Code Here

Examples of org.apache.hadoop.io.IntWritable

    private final int maxSlots;
    private List<TaskInProgress> tasksToLaunch;


    public TaskLauncher(TaskType taskType, int numSlots) {
      this.maxSlots = numSlots;
      this.numFreeSlots = new IntWritable(numSlots);
      this.tasksToLaunch = new LinkedList<TaskInProgress>();
      setDaemon(true);
      setName("TaskLauncher for " + taskType + " tasks");
    }

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.