Examples of eu.stratosphere.nephele.services.iomanager.ChannelWriterOutputView

Package eu.stratosphere.nephele.services.iomanager

Examples of eu.stratosphere.nephele.services.iomanager.ChannelWriterOutputView

eu.stratosphere.nephele.services.iomanager.ChannelWriterOutputView
A {@link DataOutputView} that is backed by a {@link BlockChannelWriter}, making it effectively a data output stream. The view writes it data in blocks to the underlying channel, adding a minimal header to each block. The data can be re-read by a {@link ChannelReaderInputView}, if it uses the same block size.

    final Channel.ID channel = this.ioManager.createChannel();
    
    // create the writer output view
    List<MemorySegment> memory = this.memoryManager.allocatePages(this.parentTask, NUM_MEMORY_SEGMENTS);
    final BlockChannelWriter writer = this.ioManager.createBlockChannelWriter(channel);
    final ChannelWriterOutputView outView = new ChannelWriterOutputView(writer, memory, MEMORY_PAGE_SIZE);
    
    // write a number of pairs
    final Record rec = new Record();
    for (int i = 0; i < NUM_PAIRS_SHORT; i++) {
      generator.next(rec);
      rec.write(outView);
    }
    this.memoryManager.release(outView.close());
    
    // create the reader input view
    memory = this.memoryManager.allocatePages(this.parentTask, NUM_MEMORY_SEGMENTS);
    final BlockChannelReader reader = this.ioManager.createBlockChannelReader(channel);
    final ChannelReaderInputView inView = new ChannelReaderInputView(reader, memory, outView.getBlockCount(), true);
    generator.reset();
    
    // read and re-generate all records and compare them
    final Record readRec = new Record();
    for (int i = 0; i < NUM_PAIRS_SHORT; i++) {

View Full Code Here

    final Channel.ID channel = this.ioManager.createChannel();
    
    // create the writer output view
    List<MemorySegment> memory = this.memoryManager.allocatePages(this.parentTask, NUM_MEMORY_SEGMENTS);
    final BlockChannelWriter writer = this.ioManager.createBlockChannelWriter(channel);
    final ChannelWriterOutputView outView = new ChannelWriterOutputView(writer, memory, MEMORY_PAGE_SIZE);
    
    // write a number of pairs
    final Record rec = new Record();
    for (int i = 0; i < NUM_PAIRS_LONG; i++) {
      generator.next(rec);
      rec.write(outView);
    }
    this.memoryManager.release(outView.close());
    
    // create the reader input view
    memory = this.memoryManager.allocatePages(this.parentTask, NUM_MEMORY_SEGMENTS);
    final BlockChannelReader reader = this.ioManager.createBlockChannelReader(channel);
    final ChannelReaderInputView inView = new ChannelReaderInputView(reader, memory, outView.getBlockCount(), true);
    generator.reset();
    
    // read and re-generate all records and compare them
    final Record readRec = new Record();
    for (int i = 0; i < NUM_PAIRS_LONG; i++) {

View Full Code Here

    final Channel.ID channel = this.ioManager.createChannel();
    
    // create the writer output view
    List<MemorySegment> memory = this.memoryManager.allocatePages(this.parentTask, NUM_MEMORY_SEGMENTS);
    final BlockChannelWriter writer = this.ioManager.createBlockChannelWriter(channel);
    final ChannelWriterOutputView outView = new ChannelWriterOutputView(writer, memory, MEMORY_PAGE_SIZE);
    
    // write a number of pairs
    final Record rec = new Record();
    for (int i = 0; i < NUM_PAIRS_SHORT; i++) {
      generator.next(rec);
      rec.write(outView);
    }
    this.memoryManager.release(outView.close());
    
    // create the reader input view
    memory = this.memoryManager.allocatePages(this.parentTask, NUM_MEMORY_SEGMENTS);
    final BlockChannelReader reader = this.ioManager.createBlockChannelReader(channel);
    final ChannelReaderInputView inView = new ChannelReaderInputView(reader, memory, outView.getBlockCount(), true);
    generator.reset();
    
    // read and re-generate all records and compare them
    try {
      final Record readRec = new Record();

View Full Code Here

    final Channel.ID channel = this.ioManager.createChannel();
    
    // create the writer output view
    List<MemorySegment> memory = this.memoryManager.allocatePages(this.parentTask, NUM_MEMORY_SEGMENTS);
    final BlockChannelWriter writer = this.ioManager.createBlockChannelWriter(channel);
    final ChannelWriterOutputView outView = new ChannelWriterOutputView(writer, memory, MEMORY_PAGE_SIZE);
    
    // write a number of pairs
    final Record rec = new Record();
    for (int i = 0; i < NUM_PAIRS_SHORT; i++) {
      generator.next(rec);
      rec.write(outView);
    }
    this.memoryManager.release(outView.close());
    
    // create the reader input view
    memory = this.memoryManager.allocatePages(this.parentTask, NUM_MEMORY_SEGMENTS);
    final BlockChannelReader reader = this.ioManager.createBlockChannelReader(channel);
    final ChannelReaderInputView inView = new ChannelReaderInputView(reader, memory, true);

View Full Code Here

    final Channel.ID channel = this.ioManager.createChannel();
    
    // create the writer output view
    List<MemorySegment> memory = this.memoryManager.allocatePages(this.parentTask, 1);
    final BlockChannelWriter writer = this.ioManager.createBlockChannelWriter(channel);
    final ChannelWriterOutputView outView = new ChannelWriterOutputView(writer, memory, MEMORY_PAGE_SIZE);
    
    // write a number of pairs
    final Record rec = new Record();
    for (int i = 0; i < NUM_PAIRS_SHORT; i++) {
      generator.next(rec);
      rec.write(outView);
    }
    this.memoryManager.release(outView.close());
    
    // create the reader input view
    memory = this.memoryManager.allocatePages(this.parentTask, 1);
    final BlockChannelReader reader = this.ioManager.createBlockChannelReader(channel);
    final ChannelReaderInputView inView = new ChannelReaderInputView(reader, memory, outView.getBlockCount(), true);
    generator.reset();
    
    // read and re-generate all records and compare them
    final Record readRec = new Record();
    for (int i = 0; i < NUM_PAIRS_SHORT; i++) {

View Full Code Here

    final Channel.ID channel = this.ioManager.createChannel();
    
    // create the writer output view
    List<MemorySegment> memory = this.memoryManager.allocatePages(this.parentTask, NUM_MEMORY_SEGMENTS);
    final BlockChannelWriter writer = this.ioManager.createBlockChannelWriter(channel);
    final ChannelWriterOutputView outView = new ChannelWriterOutputView(writer, memory, MEMORY_PAGE_SIZE);
    
    // write a number of pairs
    final Record rec = new Record();
    for (int i = 0; i < NUM_PAIRS_SHORT; i++) {
      generator.next(rec);
      rec.write(outView);
    }
    this.memoryManager.release(outView.close());
    
    // create the reader input view
    memory = this.memoryManager.allocatePages(this.parentTask, NUM_MEMORY_SEGMENTS);
    final BlockChannelReader reader = this.ioManager.createBlockChannelReader(channel);
    final ChannelReaderInputView inView = new ChannelReaderInputView(reader, memory, outView.getBlockCount(), true);
    generator.reset();
    
    // read and re-generate all records and compare them
    final Record readRec = new Record();
    for (int i = 0; i < NUM_PAIRS_SHORT / 2; i++) {

View Full Code Here

      
      // create the channel for the probe side and claim one buffer for it
      this.probeSideChannel = ioAccess.createBlockChannelWriter(probeChannelEnumerator.next(), bufferReturnQueue);
      // creating the ChannelWriterOutputView without memory will cause it to draw one segment from the
      // write behind queue, which is the spare segment we had above.
      this.probeSideBuffer = new ChannelWriterOutputView(this.probeSideChannel, this.memorySegmentSize);
    }
  }

View Full Code Here

    if (isInMemory()) {
      return;
    }
    // ATTENTION: The following lines are duplicated code from finalizeBuildPhase
    this.probeSideChannel = ioAccess.createBlockChannelWriter(probeChannelEnumerator.next(), bufferReturnQueue);
    this.probeSideBuffer = new ChannelWriterOutputView(this.probeSideChannel, this.memorySegmentSize);
  }

View Full Code Here


        // create writer
        final BlockChannelWriter writer = this.ioManager.createBlockChannelWriter(
                                channel, this.numWriteBuffersToCluster);
        registerOpenChannelToBeRemovedAtShudown(writer);
        final ChannelWriterOutputView output = new ChannelWriterOutputView(writer, this.writeMemory,
                                      this.memManager.getPageSize());


        // write sort-buffer to channel
        if (LOG.isDebugEnabled()) {
          LOG.debug("Spilling buffer " + element.id + ".");
        }
        element.buffer.writeToOutput(output);
        if (LOG.isDebugEnabled()) {
          LOG.debug("Spilled buffer " + element.id + ".");
        }


        output.close();
        unregisterOpenChannelToBeRemovedAtShudown(writer);
        
        channelIDs.add(new ChannelWithBlockCount(channel, output.getBlockCount()));


        // pass empty sort-buffer to reading thread
        element.buffer.reset();
        this.queues.empty.add(element);
      }

View Full Code Here

      final Channel.ID mergedChannelID = this.ioManager.createChannel();
      registerChannelToBeRemovedAtShudown(mergedChannelID);
      final BlockChannelWriter writer = this.ioManager.createBlockChannelWriter(
                              mergedChannelID, this.numWriteBuffersToCluster);
      registerOpenChannelToBeRemovedAtShudown(writer);
      final ChannelWriterOutputView output = new ChannelWriterOutputView(writer, writeBuffers, 
                                      this.memManager.getPageSize());


      // read the merged stream and write the data back
      final TypeSerializer<E> serializer = this.serializer;
      E rec = serializer.createInstance();
      while ((rec = mergeIterator.next(rec)) != null) {
        serializer.serialize(rec, output);
      }
      output.close();
      final int numBlocksWritten = output.getBlockCount();
      
      // register merged result to be removed at shutdown
      unregisterOpenChannelToBeRemovedAtShudown(writer);
      
      // remove the merged channel readers from the clear-at-shutdown list

View Full Code Here

0 1

TOP

Related Classes of eu.stratosphere.nephele.services.iomanager.ChannelWriterOutputView

eu.stratosphere.core.memory.MemorySegment

eu.stratosphere.pact.runtime.hash.HashPartition

eu.stratosphere.pact.runtime.io.ChannelViewsTest

eu.stratosphere.pact.runtime.sort.CombiningUnilateralSortMerger$CombiningSpillingThread

eu.stratosphere.pact.runtime.sort.UnilateralSortMerger$SpillingThread

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.