Package org.apache.flink.runtime.io.disk

Examples of org.apache.flink.runtime.io.disk.SpillingBuffer


    final TestData.Generator generator = new TestData.Generator(SEED, KEY_MAX, VALUE_LENGTH, KeyMode.RANDOM, ValueMode.RANDOM_LENGTH);
   
    // create the writer output view
    final ArrayList<MemorySegment> memory = new ArrayList<MemorySegment>(NUM_MEMORY_SEGMENTS);
    this.memoryManager.allocatePages(this.parentTask, memory, NUM_MEMORY_SEGMENTS);
    final SpillingBuffer outView = new SpillingBuffer(this.ioManager,
              new ListMemorySegmentSource(memory), this.memoryManager.getPageSize());
   
    // write a number of pairs
    final Record rec = new Record();
    for (int i = 0; i < NUM_PAIRS_INMEM; i++) {
      generator.next(rec);
      rec.write(outView);
    }
   
    // create the reader input view
    DataInputView inView = outView.flip();
    generator.reset();
   
    // read and re-generate all records and compare them
    final Record readRec = new Record();
    for (int i = 0; i < NUM_PAIRS_INMEM; i++) {
      generator.next(rec);
      readRec.read(inView);
     
      Key k1 = rec.getField(0, Key.class);
      Value v1 = rec.getField(1, Value.class);
     
      Key k2 = readRec.getField(0, Key.class);
      Value v2 = readRec.getField(1, Value.class);
     
      Assert.assertTrue("The re-generated and the read record do not match.", k1.equals(k2) && v1.equals(v2));
    }
   
    // re-read the data
    inView = outView.flip();
    generator.reset();
   
    // read and re-generate all records and compare them
    for (int i = 0; i < NUM_PAIRS_INMEM; i++) {
      generator.next(rec);
      readRec.read(inView);
     
      Key k1 = rec.getField(0, Key.class);
      Value v1 = rec.getField(1, Value.class);
     
      Key k2 = readRec.getField(0, Key.class);
      Value v2 = readRec.getField(1, Value.class);
     
      Assert.assertTrue("The re-generated and the read record do not match.", k1.equals(k2) && v1.equals(v2));
    }
   
    this.memoryManager.release(outView.close());
    this.memoryManager.release(memory);
  }
View Full Code Here


    final TestData.Generator generator = new TestData.Generator(SEED, KEY_MAX, VALUE_LENGTH, KeyMode.RANDOM, ValueMode.RANDOM_LENGTH);
   
    // create the writer output view
    final ArrayList<MemorySegment> memory = new ArrayList<MemorySegment>(NUM_MEMORY_SEGMENTS);
    this.memoryManager.allocatePages(this.parentTask, memory, NUM_MEMORY_SEGMENTS);
    final SpillingBuffer outView = new SpillingBuffer(this.ioManager,
              new ListMemorySegmentSource(memory), this.memoryManager.getPageSize());
   
    // write a number of pairs
    final Record rec = new Record();
    for (int i = 0; i < NUM_PAIRS_INMEM; i++) {
      generator.next(rec);
      rec.write(outView);
    }
   
    // create the reader input view
    DataInputView inView = outView.flip();
    generator.reset();
   
    // read and re-generate all records and compare them
    final Record readRec = new Record();
    try {
      for (int i = 0; i < NUM_PAIRS_INMEM + 1; i++) {
        generator.next(rec);
        readRec.read(inView);
       
        Key k1 = rec.getField(0, Key.class);
        Value v1 = rec.getField(1, Value.class);
       
        Key k2 = readRec.getField(0, Key.class);
        Value v2 = readRec.getField(1, Value.class);
       
        Assert.assertTrue("The re-generated and the read record do not match.", k1.equals(k2) && v1.equals(v2));
      }
      Assert.fail("Read too much, expected EOFException.");
    }
    catch (EOFException eofex) {
      // expected
    }
   
    // re-read the data
    inView = outView.flip();
    generator.reset();
   
    // read and re-generate all records and compare them
    for (int i = 0; i < NUM_PAIRS_INMEM; i++) {
      generator.next(rec);
      readRec.read(inView);
     
      Key k1 = rec.getField(0, Key.class);
      Value v1 = rec.getField(1, Value.class);
     
      Key k2 = readRec.getField(0, Key.class);
      Value v2 = readRec.getField(1, Value.class);
     
      Assert.assertTrue("The re-generated and the read record do not match.", k1.equals(k2) && v1.equals(v2));
    }
   
    this.memoryManager.release(outView.close());
    this.memoryManager.release(memory);
  }
View Full Code Here

    final TestData.Generator generator = new TestData.Generator(SEED, KEY_MAX, VALUE_LENGTH, KeyMode.RANDOM, ValueMode.RANDOM_LENGTH);
   
    // create the writer output view
    final ArrayList<MemorySegment> memory = new ArrayList<MemorySegment>(NUM_MEMORY_SEGMENTS);
    this.memoryManager.allocatePages(this.parentTask, memory, NUM_MEMORY_SEGMENTS);
    final SpillingBuffer outView = new SpillingBuffer(this.ioManager,
              new ListMemorySegmentSource(memory), this.memoryManager.getPageSize());
   
    // write a number of pairs
    final Record rec = new Record();
    for (int i = 0; i < NUM_PAIRS_EXTERNAL; i++) {
      generator.next(rec);
      rec.write(outView);
    }
   
    // create the reader input view
    DataInputView inView = outView.flip();
    generator.reset();
   
    // read and re-generate all records and compare them
    final Record readRec = new Record();
    for (int i = 0; i < NUM_PAIRS_EXTERNAL; i++) {
      generator.next(rec);
      readRec.read(inView);
     
      Key k1 = rec.getField(0, Key.class);
      Value v1 = rec.getField(1, Value.class);
     
      Key k2 = readRec.getField(0, Key.class);
      Value v2 = readRec.getField(1, Value.class);
     
      Assert.assertTrue("The re-generated and the read record do not match.", k1.equals(k2) && v1.equals(v2));
    }
   
    // re-read the data
    inView = outView.flip();
    generator.reset();
   
    // read and re-generate all records and compare them
    for (int i = 0; i < NUM_PAIRS_EXTERNAL; i++) {
      generator.next(rec);
      readRec.read(inView);
     
      Key k1 = rec.getField(0, Key.class);
      Value v1 = rec.getField(1, Value.class);
     
      Key k2 = readRec.getField(0, Key.class);
      Value v2 = readRec.getField(1, Value.class);
     
      Assert.assertTrue("The re-generated and the read record do not match.", k1.equals(k2) && v1.equals(v2));
    }
   
    this.memoryManager.release(outView.close());
    this.memoryManager.release(memory);
  }
View Full Code Here

    final TestData.Generator generator = new TestData.Generator(SEED, KEY_MAX, VALUE_LENGTH, KeyMode.RANDOM, ValueMode.RANDOM_LENGTH);
   
    // create the writer output view
    final ArrayList<MemorySegment> memory = new ArrayList<MemorySegment>(NUM_MEMORY_SEGMENTS);
    this.memoryManager.allocatePages(this.parentTask, memory, NUM_MEMORY_SEGMENTS);
    final SpillingBuffer outView = new SpillingBuffer(this.ioManager,
              new ListMemorySegmentSource(memory), this.memoryManager.getPageSize());
   
    // write a number of pairs
    final Record rec = new Record();
    for (int i = 0; i < NUM_PAIRS_EXTERNAL; i++) {
      generator.next(rec);
      rec.write(outView);
    }
   
    // create the reader input view
    DataInputView inView = outView.flip();
    generator.reset();
   
    // read and re-generate all records and compare them
    final Record readRec = new Record();
    try {
      for (int i = 0; i < NUM_PAIRS_EXTERNAL + 1; i++) {
        generator.next(rec);
        readRec.read(inView);
       
        Key k1 = rec.getField(0, Key.class);
        Value v1 = rec.getField(1, Value.class);
       
        Key k2 = readRec.getField(0, Key.class);
        Value v2 = readRec.getField(1, Value.class);
       
        Assert.assertTrue("The re-generated and the read record do not match.", k1.equals(k2) && v1.equals(v2));
      }
      Assert.fail("Read too much, expected EOFException.");
    }
    catch (EOFException eofex) {
      // expected
    }
   
    // re-read the data
    inView = outView.flip();
    generator.reset();
   
    // read and re-generate all records and compare them
    for (int i = 0; i < NUM_PAIRS_EXTERNAL; i++) {
      generator.next(rec);
      readRec.read(inView);
     
      Key k1 = rec.getField(0, Key.class);
      Value v1 = rec.getField(1, Value.class);
     
      Key k2 = readRec.getField(0, Key.class);
      Value v2 = readRec.getField(1, Value.class);
     
      Assert.assertTrue("The re-generated and the read record do not match.", k1.equals(k2) && v1.equals(v2));
    }
   
    this.memoryManager.release(outView.close());
    this.memoryManager.release(memory);
  }
View Full Code Here

    this.memManager = memManager;
   
    this.memory = new ArrayList<MemorySegment>(numPages);
    memManager.allocatePages(owner, this.memory, numPages);
   
    this.buffer = new SpillingBuffer(ioManager, new ListMemorySegmentSource(this.memory), memManager.getPageSize());
    this.tempWriter = new TempWritingThread(input, serializerFactory.getSerializer(), this.buffer);
  }
View Full Code Here

    }
   
    public void run() {
      final MutableObjectIterator<T> input = this.input;
      final TypeSerializer<T> serializer = this.serializer;
      final SpillingBuffer buffer = this.buffer;
     
      try {
        T record = serializer.createInstance();
       
        while (this.running && ((record = input.next(record)) != null)) {
View Full Code Here

   
    if (LOG.isDebugEnabled()) {
      LOG.debug("Creating spilling resettable iterator with " + memory.size() + " pages of memory.");
    }
   
    this.buffer = new SpillingBuffer(ioManager, new ListMemorySegmentSource(memory), memoryManager.getPageSize());
  }
View Full Code Here

   
    if (LOG.isDebugEnabled()) {
      LOG.debug("Creating spilling resettable iterator with " + memory.size() + " pages of memory.");
    }
   
    this.buffer = new SpillingBuffer(ioManager, new ListMemorySegmentSource(memory), memoryManager.getPageSize());
  }
View Full Code Here

TOP

Related Classes of org.apache.flink.runtime.io.disk.SpillingBuffer

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.