Examples of org.apache.flink.runtime.io.disk.iomanager.BlockChannelReader

Package org.apache.flink.runtime.io.disk.iomanager

Examples of org.apache.flink.runtime.io.disk.iomanager.BlockChannelReader

org.apache.flink.runtime.io.disk.iomanager.BlockChannelReader
A reader that reads data in blocks from a file channel. The reader reads the blocks into a {@link org.apache.flink.core.memory.MemorySegment} in an asynchronous fashion. That is, a readrequest is not processed by the thread that issues it, but by an asynchronous reader thread. Once the read request is done, the asynchronous reader adds the full MemorySegment to a return queue where it can be popped by the worker thread, once it needs the data. The return queue is in this case a {@link java.util.concurrent.LinkedBlockingQueue}, such that the working thread blocks until the request has been served, if the request is still pending when the it requires the data.
Typical pre-fetching reads are done by issuing the read requests early and popping the return queue once the data is actually needed.
The reader has no notion whether the size of the memory segments is actually the size of the blocks on disk, or even whether the file was written in blocks of the same size, or in blocks at all. Ensuring that the writing and reading is consistent with each other (same blocks sizes) is up to the programmer.

    }
    this.memoryManager.release(outView.close());
    
    // create the reader input view
    memory = this.memoryManager.allocatePages(this.parentTask, NUM_MEMORY_SEGMENTS);
    final BlockChannelReader reader = this.ioManager.createBlockChannelReader(channel);
    final ChannelReaderInputView inView = new ChannelReaderInputView(reader, memory, outView.getBlockCount(), true);
    generator.reset();
    
    // read and re-generate all records and compare them
    final Record readRec = new Record();
    for (int i = 0; i < NUM_PAIRS_SHORT; i++) {
      generator.next(rec);
      readRec.read(inView);
      
      Key k1 = rec.getField(0, Key.class);
      Value v1 = rec.getField(1, Value.class);
      
      Key k2 = readRec.getField(0, Key.class);
      Value v2 = readRec.getField(1, Value.class);
      
      Assert.assertTrue("The re-generated and the read record do not match.", k1.equals(k2) && v1.equals(v2));
    }
    
    this.memoryManager.release(inView.close());
    reader.deleteChannel();
  }

View Full Code Here

    }
    this.memoryManager.release(outView.close());
    
    // create the reader input view
    memory = this.memoryManager.allocatePages(this.parentTask, NUM_MEMORY_SEGMENTS);
    final BlockChannelReader reader = this.ioManager.createBlockChannelReader(channel);
    final ChannelReaderInputView inView = new ChannelReaderInputView(reader, memory, outView.getBlockCount(), true);
    generator.reset();
    
    // read and re-generate all records and compare them
    final Record readRec = new Record();
    for (int i = 0; i < NUM_PAIRS_LONG; i++) {
      generator.next(rec);
      readRec.read(inView);
      final Key k1 = rec.getField(0, Key.class);
      final Value v1 = rec.getField(1, Value.class);
      final Key k2 = readRec.getField(0, Key.class);
      final Value v2 = readRec.getField(1, Value.class);
      Assert.assertTrue("The re-generated and the read record do not match.", k1.equals(k2) && v1.equals(v2));
    }
    
    this.memoryManager.release(inView.close());
    reader.deleteChannel();
  }

View Full Code Here

    }
    this.memoryManager.release(outView.close());


    // create the reader input view
    memory = this.memoryManager.allocatePages(this.parentTask, NUM_MEMORY_SEGMENTS);
    final BlockChannelReader reader = this.ioManager.createBlockChannelReader(channel);
    final ChannelReaderInputView inView = new ChannelReaderInputView(reader, memory, outView.getBlockCount(), true);
    generator.reset();


    // read and re-generate all records and compare them
    try {
      final Record readRec = new Record();
      for (int i = 0; i < NUM_PAIRS_SHORT + 1; i++) {
        generator.next(rec);
        readRec.read(inView);
        final Key k1 = rec.getField(0, Key.class);
        final Value v1 = rec.getField(1, Value.class);
        final Key k2 = readRec.getField(0, Key.class);
        final Value v2 = readRec.getField(1, Value.class);
        Assert.assertTrue("The re-generated and the read record do not match.", k1.equals(k2) && v1.equals(v2));
      }
      Assert.fail("Expected an EOFException which did not occur.");
    }
    catch (EOFException eofex) {
      // expected
    }
    catch (Throwable t) {
      // unexpected
      Assert.fail("Unexpected Exception: " + t.getMessage());
    }
    
    this.memoryManager.release(inView.close());
    reader.deleteChannel();
  }

View Full Code Here

    }
    this.memoryManager.release(outView.close());
    
    // create the reader input view
    memory = this.memoryManager.allocatePages(this.parentTask, NUM_MEMORY_SEGMENTS);
    final BlockChannelReader reader = this.ioManager.createBlockChannelReader(channel);
    final ChannelReaderInputView inView = new ChannelReaderInputView(reader, memory, true);
    generator.reset();
    
    // read and re-generate all records and cmpare them
    final Record readRec = new Record();
    for (int i = 0; i < NUM_PAIRS_SHORT; i++) {
      generator.next(rec);
      readRec.read(inView);
      
      Key k1 = rec.getField(0, Key.class);
      Value v1 = rec.getField(1, Value.class);
      
      Key k2 = readRec.getField(0, Key.class);
      Value v2 = readRec.getField(1, Value.class);
      
      Assert.assertTrue("The re-generated and the read record do not match.", k1.equals(k2) && v1.equals(v2));
    }
    
    this.memoryManager.release(inView.close());
    reader.deleteChannel();
  }

View Full Code Here

    }
    this.memoryManager.release(outView.close());
    
    // create the reader input view
    memory = this.memoryManager.allocatePages(this.parentTask, 1);
    final BlockChannelReader reader = this.ioManager.createBlockChannelReader(channel);
    final ChannelReaderInputView inView = new ChannelReaderInputView(reader, memory, outView.getBlockCount(), true);
    generator.reset();
    
    // read and re-generate all records and compare them
    final Record readRec = new Record();
    for (int i = 0; i < NUM_PAIRS_SHORT; i++) {
      generator.next(rec);
      readRec.read(inView);
      
      Key k1 = rec.getField(0, Key.class);
      Value v1 = rec.getField(1, Value.class);
      
      Key k2 = readRec.getField(0, Key.class);
      Value v2 = readRec.getField(1, Value.class);
      
      Assert.assertTrue("The re-generated and the read record do not match.", k1.equals(k2) && v1.equals(v2));
    }
    
    this.memoryManager.release(inView.close());
    reader.deleteChannel();
  }

View Full Code Here

    }
    this.memoryManager.release(outView.close());
    
    // create the reader input view
    memory = this.memoryManager.allocatePages(this.parentTask, NUM_MEMORY_SEGMENTS);
    final BlockChannelReader reader = this.ioManager.createBlockChannelReader(channel);
    final ChannelReaderInputView inView = new ChannelReaderInputView(reader, memory, outView.getBlockCount(), true);
    generator.reset();
    
    // read and re-generate all records and compare them
    final Record readRec = new Record();
    for (int i = 0; i < NUM_PAIRS_SHORT / 2; i++) {
      generator.next(rec);
      readRec.read(inView);
      
      Key k1 = rec.getField(0, Key.class);
      Value v1 = rec.getField(1, Value.class);
      
      Key k2 = readRec.getField(0, Key.class);
      Value v2 = readRec.getField(1, Value.class);
      
      Assert.assertTrue("The re-generated and the read record do not match.", k1.equals(k2) && v1.equals(v2));
    }
    
    this.memoryManager.release(inView.close());
    reader.deleteChannel();
  }

View Full Code Here

  {
    final List<MemorySegment> memory = this.memManager.allocatePages(memoryOwner, numSegments);
    final Channel.ID channel = this.ioManager.createChannel();
    
    BlockChannelWriter writer = null;
    BlockChannelReader reader = null;
    
    try {  
      writer = this.ioManager.createBlockChannelWriter(channel);
      final ChannelWriterOutputView out = new ChannelWriterOutputView(writer, memory, this.memManager.getPageSize());
      
      long writeStart = System.currentTimeMillis();
      
      int valsLeft = NUM_INTS_WRITTEN;
      while (valsLeft-- > 0) {
        out.writeInt(valsLeft);
      }
      
      out.close();
      final int numBlocks = out.getBlockCount();
      writer.close();
      writer = null;
      
      long writeElapsed = System.currentTimeMillis() - writeStart;
      
      // ----------------------------------------------------------------
      
      reader = ioManager.createBlockChannelReader(channel);
      final ChannelReaderInputView in = new ChannelReaderInputView(reader, memory, numBlocks, false);
      
      long readStart = System.currentTimeMillis();
      
      valsLeft = NUM_INTS_WRITTEN;
      while (valsLeft-- > 0) {
        in.readInt();
//        Assert.assertTrue(rec.getValue() == valsLeft);
      }
      
      in.close();
      reader.close();
      
      long readElapsed = System.currentTimeMillis() - readStart;
      
      reader.deleteChannel();
      reader = null;
      
      LOG.info("IOManager with " + numSegments + " mem segments: write " + writeElapsed + " msecs, read " + readElapsed + " msecs.");
      
      memManager.release(memory);
    }
    finally {
      if (reader != null) {
        reader.closeAndDelete();
      }
      if (writer != null) {
        writer.closeAndDelete();
      }
    }

View Full Code Here

        memSeg = writer.getNextReturnedSegment();
      }
      
      writer.close();
      
      final BlockChannelReader reader = this.ioManager.createBlockChannelReader(channelID);
      for (int i = 0; i < NUM_IOS; i++) {
        reader.readBlock(memSeg);
        memSeg = reader.getNextReturnedSegment();
        
        for (int pos = 0; pos < memSeg.size(); pos += 4) {
          if (memSeg.getInt(pos) != i) {
            Assert.fail("Read memory segment contains invalid data.");
          }
        }
      }
      
      reader.closeAndDelete();
      
      this.memoryManager.release(memSeg);
      
    } catch (Exception ex) {
      ex.printStackTrace();

View Full Code Here

      // get back the memory
      while (memSegs.size() < NUM_SEGS) {
        memSegs.add(writer.getNextReturnedSegment());
      }
      
      final BlockChannelReader reader = this.ioManager.createBlockChannelReader(channelID);
      while(!memSegs.isEmpty()) {
        reader.readBlock(memSegs.remove(0));
      }
      
      for (int i = 0; i < NUM_IOS; i++) {
        final MemorySegment memSeg = reader.getNextReturnedSegment();
        
        for (int pos = 0; pos < memSeg.size(); pos += 4) {
          if (memSeg.getInt(pos) != i) {
            Assert.fail("Read memory segment contains invalid data.");
          }
        }
        reader.readBlock(memSeg);
      }
      
      reader.closeAndDelete();
      
      // get back the memory
      while (memSegs.size() < NUM_SEGS) {
        memSegs.add(reader.getNextReturnedSegment());
      }
      
      this.memoryManager.release(memSegs);
      
    } catch (Exception ex) {

View Full Code Here

      for (int i = 0; i < channelIDs.size(); i++) {
        final ChannelWithBlockCount channel = channelIDs.get(i);
        final List<MemorySegment> segsForChannel = inputSegments.get(i);
        
        // create a reader. if there are multiple segments for the reader, issue multiple together per I/O request
        final BlockChannelReader reader = segsForChannel.size() >= 4 ? 
          this.ioManager.createBlockChannelReader(channel.getChannel(), segsForChannel.size() / 2) :
          this.ioManager.createBlockChannelReader(channel.getChannel());
          
        readerList.add(reader);
        registerOpenChannelToBeRemovedAtShudown(reader);

View Full Code Here

0 1

TOP

Related Classes of org.apache.flink.runtime.io.disk.iomanager.BlockChannelReader

org.apache.flink.core.memory.MemorySegment

org.apache.flink.runtime.io.disk.ChannelViewsTest

org.apache.flink.runtime.io.disk.iomanager.IOManagerPerformanceBenchmark

org.apache.flink.runtime.io.disk.iomanager.IOManagerTest

org.apache.flink.runtime.io.disk.SpillingBuffer

org.apache.flink.runtime.iterative.io.SerializedUpdateBuffer

org.apache.flink.runtime.operators.hash.MutableHashTable

org.apache.flink.runtime.operators.sort.UnilateralSortMerger$SpillingThread

java.io.IOException

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.