Examples of org.apache.flink.runtime.io.disk.iomanager.ChannelReaderInputView

Package org.apache.flink.runtime.io.disk.iomanager

Examples of org.apache.flink.runtime.io.disk.iomanager.ChannelReaderInputView

org.apache.flink.runtime.io.disk.iomanager.ChannelReaderInputView
A {@link org.apache.flink.core.memory.DataInputView} that is backed by a{@link BlockChannelReader}, making it effectively a data input stream. The view reads it data in blocks from the underlying channel. The view can only read data that has been written by a {@link ChannelWriterOutputView}, due to block formatting.

    this.memoryManager.release(outView.close());
    
    // create the reader input view
    memory = this.memoryManager.allocatePages(this.parentTask, NUM_MEMORY_SEGMENTS);
    final BlockChannelReader reader = this.ioManager.createBlockChannelReader(channel);
    final ChannelReaderInputView inView = new ChannelReaderInputView(reader, memory, outView.getBlockCount(), true);
    generator.reset();
    
    // read and re-generate all records and compare them
    final Record readRec = new Record();
    for (int i = 0; i < NUM_PAIRS_SHORT; i++) {
      generator.next(rec);
      readRec.read(inView);
      
      Key k1 = rec.getField(0, Key.class);
      Value v1 = rec.getField(1, Value.class);
      
      Key k2 = readRec.getField(0, Key.class);
      Value v2 = readRec.getField(1, Value.class);
      
      Assert.assertTrue("The re-generated and the read record do not match.", k1.equals(k2) && v1.equals(v2));
    }
    
    this.memoryManager.release(inView.close());
    reader.deleteChannel();
  }

View Full Code Here

    this.memoryManager.release(outView.close());
    
    // create the reader input view
    memory = this.memoryManager.allocatePages(this.parentTask, NUM_MEMORY_SEGMENTS);
    final BlockChannelReader reader = this.ioManager.createBlockChannelReader(channel);
    final ChannelReaderInputView inView = new ChannelReaderInputView(reader, memory, outView.getBlockCount(), true);
    generator.reset();
    
    // read and re-generate all records and compare them
    final Record readRec = new Record();
    for (int i = 0; i < NUM_PAIRS_LONG; i++) {
      generator.next(rec);
      readRec.read(inView);
      final Key k1 = rec.getField(0, Key.class);
      final Value v1 = rec.getField(1, Value.class);
      final Key k2 = readRec.getField(0, Key.class);
      final Value v2 = readRec.getField(1, Value.class);
      Assert.assertTrue("The re-generated and the read record do not match.", k1.equals(k2) && v1.equals(v2));
    }
    
    this.memoryManager.release(inView.close());
    reader.deleteChannel();
  }

View Full Code Here

    this.memoryManager.release(outView.close());


    // create the reader input view
    memory = this.memoryManager.allocatePages(this.parentTask, NUM_MEMORY_SEGMENTS);
    final BlockChannelReader reader = this.ioManager.createBlockChannelReader(channel);
    final ChannelReaderInputView inView = new ChannelReaderInputView(reader, memory, outView.getBlockCount(), true);
    generator.reset();


    // read and re-generate all records and compare them
    try {
      final Record readRec = new Record();
      for (int i = 0; i < NUM_PAIRS_SHORT + 1; i++) {
        generator.next(rec);
        readRec.read(inView);
        final Key k1 = rec.getField(0, Key.class);
        final Value v1 = rec.getField(1, Value.class);
        final Key k2 = readRec.getField(0, Key.class);
        final Value v2 = readRec.getField(1, Value.class);
        Assert.assertTrue("The re-generated and the read record do not match.", k1.equals(k2) && v1.equals(v2));
      }
      Assert.fail("Expected an EOFException which did not occur.");
    }
    catch (EOFException eofex) {
      // expected
    }
    catch (Throwable t) {
      // unexpected
      Assert.fail("Unexpected Exception: " + t.getMessage());
    }
    
    this.memoryManager.release(inView.close());
    reader.deleteChannel();
  }

View Full Code Here

    this.memoryManager.release(outView.close());
    
    // create the reader input view
    memory = this.memoryManager.allocatePages(this.parentTask, NUM_MEMORY_SEGMENTS);
    final BlockChannelReader reader = this.ioManager.createBlockChannelReader(channel);
    final ChannelReaderInputView inView = new ChannelReaderInputView(reader, memory, true);
    generator.reset();
    
    // read and re-generate all records and cmpare them
    final Record readRec = new Record();
    for (int i = 0; i < NUM_PAIRS_SHORT; i++) {
      generator.next(rec);
      readRec.read(inView);
      
      Key k1 = rec.getField(0, Key.class);
      Value v1 = rec.getField(1, Value.class);
      
      Key k2 = readRec.getField(0, Key.class);
      Value v2 = readRec.getField(1, Value.class);
      
      Assert.assertTrue("The re-generated and the read record do not match.", k1.equals(k2) && v1.equals(v2));
    }
    
    this.memoryManager.release(inView.close());
    reader.deleteChannel();
  }

View Full Code Here

    this.memoryManager.release(outView.close());
    
    // create the reader input view
    memory = this.memoryManager.allocatePages(this.parentTask, 1);
    final BlockChannelReader reader = this.ioManager.createBlockChannelReader(channel);
    final ChannelReaderInputView inView = new ChannelReaderInputView(reader, memory, outView.getBlockCount(), true);
    generator.reset();
    
    // read and re-generate all records and compare them
    final Record readRec = new Record();
    for (int i = 0; i < NUM_PAIRS_SHORT; i++) {
      generator.next(rec);
      readRec.read(inView);
      
      Key k1 = rec.getField(0, Key.class);
      Value v1 = rec.getField(1, Value.class);
      
      Key k2 = readRec.getField(0, Key.class);
      Value v2 = readRec.getField(1, Value.class);
      
      Assert.assertTrue("The re-generated and the read record do not match.", k1.equals(k2) && v1.equals(v2));
    }
    
    this.memoryManager.release(inView.close());
    reader.deleteChannel();
  }

View Full Code Here

    this.memoryManager.release(outView.close());
    
    // create the reader input view
    memory = this.memoryManager.allocatePages(this.parentTask, NUM_MEMORY_SEGMENTS);
    final BlockChannelReader reader = this.ioManager.createBlockChannelReader(channel);
    final ChannelReaderInputView inView = new ChannelReaderInputView(reader, memory, outView.getBlockCount(), true);
    generator.reset();
    
    // read and re-generate all records and compare them
    final Record readRec = new Record();
    for (int i = 0; i < NUM_PAIRS_SHORT / 2; i++) {
      generator.next(rec);
      readRec.read(inView);
      
      Key k1 = rec.getField(0, Key.class);
      Value v1 = rec.getField(1, Value.class);
      
      Key k2 = readRec.getField(0, Key.class);
      Value v2 = readRec.getField(1, Value.class);
      
      Assert.assertTrue("The re-generated and the read record do not match.", k1.equals(k2) && v1.equals(v2));
    }
    
    this.memoryManager.release(inView.close());
    reader.deleteChannel();
  }

View Full Code Here

      long writeElapsed = System.currentTimeMillis() - writeStart;
      
      // ----------------------------------------------------------------
      
      reader = ioManager.createBlockChannelReader(channel);
      final ChannelReaderInputView in = new ChannelReaderInputView(reader, memory, numBlocks, false);
      
      long readStart = System.currentTimeMillis();
      
      valsLeft = NUM_INTS_WRITTEN;
      while (valsLeft-- > 0) {
        in.readInt();
//        Assert.assertTrue(rec.getValue() == valsLeft);
      }
      
      in.close();
      reader.close();
      
      long readElapsed = System.currentTimeMillis() - readStart;
      
      reader.deleteChannel();

View Full Code Here

        readerList.add(reader);
        registerOpenChannelToBeRemovedAtShudown(reader);
        unregisterChannelToBeRemovedAtShudown(channel.getChannel());
        
        // wrap channel reader as a view, to get block spanning record deserialization
        final ChannelReaderInputView inView = new ChannelReaderInputView(reader, segsForChannel, 
                                      channel.getBlockCount(), false);
        iterators.add(new ChannelReaderInputViewIterator<E>(inView, null, this.serializer));
      }


      return new MergeIterator<E>(iterators, this.serializer, this.comparator);

View Full Code Here

      List<MemorySegment> segments, List<MemorySegment> freeMemTarget, TypeSerializer<E> accessors, int numBlocks)
  throws IOException
  {
    this.accessors = accessors;
    this.freeMemTarget = freeMemTarget;
    this.inView = new ChannelReaderInputView(reader, segments, numBlocks, false);
  }

View Full Code Here

      final List<MemorySegment> segments = new ArrayList<MemorySegment>(2);
      segments.add(getNextBuffer());
      segments.add(getNextBuffer());
      
      final BlockChannelReader inReader = this.ioManager.createBlockChannelReader(p.getBuildSideChannel().getChannelID());
      final ChannelReaderInputView inView = new HeaderlessChannelReaderInputView(inReader, segments,
            p.getBuildSideBlockCount(), p.getLastSegmentLimit(), false);
      final ChannelReaderInputViewIterator<BT> inIter = new ChannelReaderInputViewIterator<BT>(inView, 
          this.availableMemory, this.buildSideSerializer);
      final TypeComparator<BT> btComparator = this.buildSideComparator;
      BT rec = this.buildSideSerializer.createInstance();

View Full Code Here

TOP

Related Classes of org.apache.flink.runtime.io.disk.iomanager.ChannelReaderInputView

org.apache.flink.core.memory.MemorySegment

org.apache.flink.runtime.io.disk.ChannelReaderInputViewIterator

org.apache.flink.runtime.io.disk.ChannelViewsTest

org.apache.flink.runtime.io.disk.iomanager.IOManagerPerformanceBenchmark

org.apache.flink.runtime.operators.hash.MutableHashTable

org.apache.flink.runtime.operators.sort.UnilateralSortMerger$SpillingThread

java.io.EOFException

java.io.IOException

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.