Examples of eu.stratosphere.types.Record

eu.stratosphere.types.Record
Record is the basic data record that flows between functions in a Stratosphere program. The record is a tuple of arbitrary values. It implements a sparse tuple model, meaning that the record can contain many fields which are actually null and not represented in the record. It has internally a bitmap marking which fields are set and which are not.
For efficient data exchange, a record that is read from any source holds its data in serialized binary form. Fields are deserialized lazily upon first access. Modified fields are cached and the modifications are incorporated into the binary representation upon the next serialization or any explicit call to the {@link #updateBinaryRepresenation()} method.
IMPORTANT NOTE: Records must be used as mutable objects and be reused across user function calls in order to achieve performance. The record is a heavy-weight object, designed to minimize calls to the individual fields' serialization and deserialization methods. It holds quite a bit of state consumes a comparably large amount of memory (> 200 bytes in a 64 bit JVM) due to several pointers and arrays.
This class is NOT thread-safe!

    // merge iterator
    MutableObjectIterator<Record> iterator = new MergeIterator<Record>(iterators, this.serializer, this.comparator);


    int elementsFound = 1;
    // check expected order
    Record rec1 = new Record();
    Record rec2 = new Record();
    final Key k1 = new Key();
    final Key k2 = new Key();
    
    Assert.assertTrue((rec1 = iterator.next(rec1)) != null);
    while ((rec2 = iterator.next(rec2)) != null) {
      elementsFound++;
      k1.setKey(rec1.getField(0, TestData.Key.class).getKey());
      k2.setKey(rec2.getField(0, TestData.Key.class).getKey());
      Assert.assertTrue(comparator.compare(k1, k2) <= 0);
      
      Record tmp = rec1;
      rec1 = rec2;
      rec2 = tmp;
    }
    
    Assert.assertEquals("Too few elements returned from stream.", 50, elementsFound);

View Full Code Here

    MutableObjectIterator<Record> iterator = new MergeIterator<Record>(iterators, this.serializer, this.comparator);


    boolean violationFound = false;
    
    // check expected order
    Record rec1 = new Record();
    Record rec2 = new Record();
    
    Assert.assertTrue((rec1 = iterator.next(rec1)) != null);
    while ((rec2 = iterator.next(rec2)) != null)
    {
      final Key k1 = new Key();
      final Key k2 = new Key();
      k1.setKey(rec1.getField(0, TestData.Key.class).getKey());
      k2.setKey(rec2.getField(0, TestData.Key.class).getKey());
      
      if (comparator.compare(k1, k2) > 0) {
        violationFound = true;
        break;
      }
      
      Record tmp = rec1;
      rec1 = rec2;
      rec2 = tmp;
    }
    
    Assert.assertTrue("Merge must have returned a wrong result", violationFound);

View Full Code Here

    List<MemorySegment> memory = this.memoryManager.allocatePages(this.parentTask, NUM_MEMORY_SEGMENTS);
    final BlockChannelWriter writer = this.ioManager.createBlockChannelWriter(channel);
    final ChannelWriterOutputView outView = new ChannelWriterOutputView(writer, memory, MEMORY_PAGE_SIZE);
    
    // write a number of pairs
    final Record rec = new Record();
    for (int i = 0; i < NUM_PAIRS_SHORT; i++) {
      generator.next(rec);
      rec.write(outView);
    }
    this.memoryManager.release(outView.close());
    
    // create the reader input view
    memory = this.memoryManager.allocatePages(this.parentTask, NUM_MEMORY_SEGMENTS);
    final BlockChannelReader reader = this.ioManager.createBlockChannelReader(channel);
    final ChannelReaderInputView inView = new ChannelReaderInputView(reader, memory, outView.getBlockCount(), true);
    generator.reset();
    
    // read and re-generate all records and compare them
    final Record readRec = new Record();
    for (int i = 0; i < NUM_PAIRS_SHORT; i++) {
      generator.next(rec);
      readRec.read(inView);
      
      Key k1 = rec.getField(0, Key.class);
      Value v1 = rec.getField(1, Value.class);
      
      Key k2 = readRec.getField(0, Key.class);
      Value v2 = readRec.getField(1, Value.class);
      
      Assert.assertTrue("The re-generated and the read record do not match.", k1.equals(k2) && v1.equals(v2));
    }
    
    this.memoryManager.release(inView.close());

View Full Code Here

    List<MemorySegment> memory = this.memoryManager.allocatePages(this.parentTask, NUM_MEMORY_SEGMENTS);
    final BlockChannelWriter writer = this.ioManager.createBlockChannelWriter(channel);
    final ChannelWriterOutputView outView = new ChannelWriterOutputView(writer, memory, MEMORY_PAGE_SIZE);
    
    // write a number of pairs
    final Record rec = new Record();
    for (int i = 0; i < NUM_PAIRS_LONG; i++) {
      generator.next(rec);
      rec.write(outView);
    }
    this.memoryManager.release(outView.close());
    
    // create the reader input view
    memory = this.memoryManager.allocatePages(this.parentTask, NUM_MEMORY_SEGMENTS);
    final BlockChannelReader reader = this.ioManager.createBlockChannelReader(channel);
    final ChannelReaderInputView inView = new ChannelReaderInputView(reader, memory, outView.getBlockCount(), true);
    generator.reset();
    
    // read and re-generate all records and compare them
    final Record readRec = new Record();
    for (int i = 0; i < NUM_PAIRS_LONG; i++) {
      generator.next(rec);
      readRec.read(inView);
      final Key k1 = rec.getField(0, Key.class);
      final Value v1 = rec.getField(1, Value.class);
      final Key k2 = readRec.getField(0, Key.class);
      final Value v2 = readRec.getField(1, Value.class);
      Assert.assertTrue("The re-generated and the read record do not match.", k1.equals(k2) && v1.equals(v2));
    }
    
    this.memoryManager.release(inView.close());
    reader.deleteChannel();

View Full Code Here

    List<MemorySegment> memory = this.memoryManager.allocatePages(this.parentTask, NUM_MEMORY_SEGMENTS);
    final BlockChannelWriter writer = this.ioManager.createBlockChannelWriter(channel);
    final ChannelWriterOutputView outView = new ChannelWriterOutputView(writer, memory, MEMORY_PAGE_SIZE);
    
    // write a number of pairs
    final Record rec = new Record();
    for (int i = 0; i < NUM_PAIRS_SHORT; i++) {
      generator.next(rec);
      rec.write(outView);
    }
    this.memoryManager.release(outView.close());
    
    // create the reader input view
    memory = this.memoryManager.allocatePages(this.parentTask, NUM_MEMORY_SEGMENTS);
    final BlockChannelReader reader = this.ioManager.createBlockChannelReader(channel);
    final ChannelReaderInputView inView = new ChannelReaderInputView(reader, memory, outView.getBlockCount(), true);
    generator.reset();
    
    // read and re-generate all records and compare them
    try {
      final Record readRec = new Record();
      for (int i = 0; i < NUM_PAIRS_SHORT + 1; i++) {
        generator.next(rec);
        readRec.read(inView);
        final Key k1 = rec.getField(0, Key.class);
        final Value v1 = rec.getField(1, Value.class);
        final Key k2 = readRec.getField(0, Key.class);
        final Value v2 = readRec.getField(1, Value.class);
        Assert.assertTrue("The re-generated and the read record do not match.", k1.equals(k2) && v1.equals(v2));
      }
      Assert.fail("Expected an EOFException which did not occur.");
    }
    catch (EOFException eofex) {

View Full Code Here

    List<MemorySegment> memory = this.memoryManager.allocatePages(this.parentTask, NUM_MEMORY_SEGMENTS);
    final BlockChannelWriter writer = this.ioManager.createBlockChannelWriter(channel);
    final ChannelWriterOutputView outView = new ChannelWriterOutputView(writer, memory, MEMORY_PAGE_SIZE);
    
    // write a number of pairs
    final Record rec = new Record();
    for (int i = 0; i < NUM_PAIRS_SHORT; i++) {
      generator.next(rec);
      rec.write(outView);
    }
    this.memoryManager.release(outView.close());
    
    // create the reader input view
    memory = this.memoryManager.allocatePages(this.parentTask, NUM_MEMORY_SEGMENTS);
    final BlockChannelReader reader = this.ioManager.createBlockChannelReader(channel);
    final ChannelReaderInputView inView = new ChannelReaderInputView(reader, memory, true);
    generator.reset();
    
    // read and re-generate all records and cmpare them
    final Record readRec = new Record();
    for (int i = 0; i < NUM_PAIRS_SHORT; i++) {
      generator.next(rec);
      readRec.read(inView);
      
      Key k1 = rec.getField(0, Key.class);
      Value v1 = rec.getField(1, Value.class);
      
      Key k2 = readRec.getField(0, Key.class);
      Value v2 = readRec.getField(1, Value.class);
      
      Assert.assertTrue("The re-generated and the read record do not match.", k1.equals(k2) && v1.equals(v2));
    }
    
    this.memoryManager.release(inView.close());

View Full Code Here

    List<MemorySegment> memory = this.memoryManager.allocatePages(this.parentTask, 1);
    final BlockChannelWriter writer = this.ioManager.createBlockChannelWriter(channel);
    final ChannelWriterOutputView outView = new ChannelWriterOutputView(writer, memory, MEMORY_PAGE_SIZE);
    
    // write a number of pairs
    final Record rec = new Record();
    for (int i = 0; i < NUM_PAIRS_SHORT; i++) {
      generator.next(rec);
      rec.write(outView);
    }
    this.memoryManager.release(outView.close());
    
    // create the reader input view
    memory = this.memoryManager.allocatePages(this.parentTask, 1);
    final BlockChannelReader reader = this.ioManager.createBlockChannelReader(channel);
    final ChannelReaderInputView inView = new ChannelReaderInputView(reader, memory, outView.getBlockCount(), true);
    generator.reset();
    
    // read and re-generate all records and compare them
    final Record readRec = new Record();
    for (int i = 0; i < NUM_PAIRS_SHORT; i++) {
      generator.next(rec);
      readRec.read(inView);
      
      Key k1 = rec.getField(0, Key.class);
      Value v1 = rec.getField(1, Value.class);
      
      Key k2 = readRec.getField(0, Key.class);
      Value v2 = readRec.getField(1, Value.class);
      
      Assert.assertTrue("The re-generated and the read record do not match.", k1.equals(k2) && v1.equals(v2));
    }
    
    this.memoryManager.release(inView.close());

View Full Code Here

    List<MemorySegment> memory = this.memoryManager.allocatePages(this.parentTask, NUM_MEMORY_SEGMENTS);
    final BlockChannelWriter writer = this.ioManager.createBlockChannelWriter(channel);
    final ChannelWriterOutputView outView = new ChannelWriterOutputView(writer, memory, MEMORY_PAGE_SIZE);
    
    // write a number of pairs
    final Record rec = new Record();
    for (int i = 0; i < NUM_PAIRS_SHORT; i++) {
      generator.next(rec);
      rec.write(outView);
    }
    this.memoryManager.release(outView.close());
    
    // create the reader input view
    memory = this.memoryManager.allocatePages(this.parentTask, NUM_MEMORY_SEGMENTS);
    final BlockChannelReader reader = this.ioManager.createBlockChannelReader(channel);
    final ChannelReaderInputView inView = new ChannelReaderInputView(reader, memory, outView.getBlockCount(), true);
    generator.reset();
    
    // read and re-generate all records and compare them
    final Record readRec = new Record();
    for (int i = 0; i < NUM_PAIRS_SHORT / 2; i++) {
      generator.next(rec);
      readRec.read(inView);
      
      Key k1 = rec.getField(0, Key.class);
      Value v1 = rec.getField(1, Value.class);
      
      Key k2 = readRec.getField(0, Key.class);
      Value v2 = readRec.getField(1, Value.class);
      
      Assert.assertTrue("The re-generated and the read record do not match.", k1.equals(k2) && v1.equals(v2));
    }
    
    this.memoryManager.release(inView.close());

View Full Code Here

    this.memman = new DefaultMemoryManager(MEMORY_CAPACITY);
    
    // create test objects
    this.objects = new ArrayList<Record>(20000);
    for (int i = 0; i < NUM_VALUES; ++i) {
      this.objects.add(new Record(new IntValue(i)));
    }
    
    // create the reader
    this.reader = objects.iterator();
  }

View Full Code Here

    do {
      lower = upper;
      upper = lower;
      // find the upper bound
      while (iterator.hasNext()) {
        Record target = iterator.next();
        int val = target.getField(0, IntValue.class).getValue();
        Assert.assertEquals(upper++, val);
      }
      // now reset the buffer a few times
      for (int i = 0; i < 5; ++i) {
        iterator.reset();
        int count = 0;
        while (iterator.hasNext()) {
          Record target = iterator.next();
          int val = target.getField(0, IntValue.class).getValue();
          Assert.assertEquals(lower + (count++), val);
        }
        Assert.assertEquals(upper - lower, count);
      }
    } while (iterator.nextBlock());

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of eu.stratosphere.types.Record

eu.stratosphere.api.avro.AvroWithEmptyArrayITCase$MyCoGrouper

eu.stratosphere.api.common.io.DelimitedInputFormatTest

eu.stratosphere.api.common.io.SequentialFormatTest

eu.stratosphere.api.java.record.io.avro.AvroRecordInputFormatTest

eu.stratosphere.api.java.record.io.avro.example.ReflectiveAvroTypeExample$ConcatenatingReducer

eu.stratosphere.api.java.record.io.CsvInputFormatTest

eu.stratosphere.api.java.record.io.CsvOutputFormatTest

eu.stratosphere.api.java.record.io.ExternalProcessFixedLengthInputFormatTest

eu.stratosphere.api.java.record.io.ExternalProcessInputFormatTest

eu.stratosphere.api.java.record.io.FixedLenghtInputFormatTest

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.