Examples of org.apache.flink.runtime.memorymanager.ListMemorySegmentSource

Package org.apache.flink.runtime.memorymanager

Examples of org.apache.flink.runtime.memorymanager.ListMemorySegmentSource

org.apache.flink.runtime.memorymanager.ListMemorySegmentSource
Simple memory segment source that draws segments from a list.

    
    // create the writer output view
    final ArrayList<MemorySegment> memory = new ArrayList<MemorySegment>(NUM_MEMORY_SEGMENTS);
    this.memoryManager.allocatePages(this.parentTask, memory, NUM_MEMORY_SEGMENTS);
    final SpillingBuffer outView = new SpillingBuffer(this.ioManager, 
              new ListMemorySegmentSource(memory), this.memoryManager.getPageSize());
    
    // write a number of pairs
    final Record rec = new Record();
    for (int i = 0; i < NUM_PAIRS_INMEM; i++) {
      generator.next(rec);

View Full Code Here

    
    // create the writer output view
    final ArrayList<MemorySegment> memory = new ArrayList<MemorySegment>(NUM_MEMORY_SEGMENTS);
    this.memoryManager.allocatePages(this.parentTask, memory, NUM_MEMORY_SEGMENTS);
    final SpillingBuffer outView = new SpillingBuffer(this.ioManager, 
              new ListMemorySegmentSource(memory), this.memoryManager.getPageSize());
    
    // write a number of pairs
    final Record rec = new Record();
    for (int i = 0; i < NUM_PAIRS_INMEM; i++) {
      generator.next(rec);

View Full Code Here

    
    // create the writer output view
    final ArrayList<MemorySegment> memory = new ArrayList<MemorySegment>(NUM_MEMORY_SEGMENTS);
    this.memoryManager.allocatePages(this.parentTask, memory, NUM_MEMORY_SEGMENTS);
    final SpillingBuffer outView = new SpillingBuffer(this.ioManager, 
              new ListMemorySegmentSource(memory), this.memoryManager.getPageSize());
    
    // write a number of pairs
    final Record rec = new Record();
    for (int i = 0; i < NUM_PAIRS_EXTERNAL; i++) {
      generator.next(rec);

View Full Code Here

    
    // create the writer output view
    final ArrayList<MemorySegment> memory = new ArrayList<MemorySegment>(NUM_MEMORY_SEGMENTS);
    this.memoryManager.allocatePages(this.parentTask, memory, NUM_MEMORY_SEGMENTS);
    final SpillingBuffer outView = new SpillingBuffer(this.ioManager, 
              new ListMemorySegmentSource(memory), this.memoryManager.getPageSize());
    
    // write a number of pairs
    final Record rec = new Record();
    for (int i = 0; i < NUM_PAIRS_EXTERNAL; i++) {
      generator.next(rec);

View Full Code Here

  // --------------------------------------------------------------------------------------------


  private void createPartitions(int numPartitions) {
    this.partitions.clear();
    
    ListMemorySegmentSource memSource = new ListMemorySegmentSource(this.availableMemory);
    this.pageSizeInBits = MathUtils.log2strict(this.segmentSize);
    
    for (int i = 0; i < numPartitions; i++) {
      this.partitions.add(new InMemoryPartition<T>(this.buildSideSerializer, i, memSource, this.segmentSize, pageSizeInBits));
    }

View Full Code Here

    this.sortIndex = new ArrayList<MemorySegment>(16);
    this.recordBufferSegments = new ArrayList<MemorySegment>(16);
    
    // the views for the record collections
    this.recordCollector = new SimpleCollectingOutputView(this.recordBufferSegments,
      new ListMemorySegmentSource(this.freeMemory), this.segmentSize);
    this.recordBuffer = new RandomAccessInputView(this.recordBufferSegments, this.segmentSize);
    this.recordBufferForComparison = new RandomAccessInputView(this.recordBufferSegments, this.segmentSize);
    
    // set up normalized key characteristics
    if (this.comparator.supportsNormalizedKey()) {

View Full Code Here

    this.memManager = memManager;
    
    this.memory = new ArrayList<MemorySegment>(numPages);
    memManager.allocatePages(owner, this.memory, numPages);
    
    this.buffer = new SpillingBuffer(ioManager, new ListMemorySegmentSource(this.memory), memManager.getPageSize());
    this.tempWriter = new TempWritingThread(input, serializerFactory.getSerializer(), this.buffer);
  }

View Full Code Here

    this.emptySegments = new ArrayList<MemorySegment>(numPages);
    this.fullSegments = new ArrayList<MemorySegment>(numPages);
    memoryManager.allocatePages(ownerTask, emptySegments, numPages);
    
    this.collectingView = new SimpleCollectingOutputView(this.fullSegments, 
            new ListMemorySegmentSource(this.emptySegments), memoryManager.getPageSize());
    this.readView = new RandomAccessInputView(this.fullSegments, memoryManager.getPageSize());
    
    if (LOG.isDebugEnabled()) {
      LOG.debug("Iterator initialized using " + numPages + " memory buffers.");
    }

View Full Code Here

    
    if (LOG.isDebugEnabled()) {
      LOG.debug("Creating spilling resettable iterator with " + memory.size() + " pages of memory.");
    }
    
    this.buffer = new SpillingBuffer(ioManager, new ListMemorySegmentSource(memory), memoryManager.getPageSize());
  }

View Full Code Here

    
    if (LOG.isDebugEnabled()) {
      LOG.debug("Creating spilling resettable iterator with " + memory.size() + " pages of memory.");
    }
    
    this.buffer = new SpillingBuffer(ioManager, new ListMemorySegmentSource(memory), memoryManager.getPageSize());
  }

View Full Code Here

TOP

Related Classes of org.apache.flink.runtime.memorymanager.ListMemorySegmentSource

org.apache.flink.runtime.io.disk.SpillingBufferTest

org.apache.flink.runtime.operators.hash.CompactingHashTable

org.apache.flink.runtime.operators.resettable.AbstractBlockResettableIterator

org.apache.flink.runtime.operators.resettable.SpillingResettableIterator

org.apache.flink.runtime.operators.resettable.SpillingResettableMutableObjectIterator

org.apache.flink.runtime.operators.sort.NormalizedKeySorter

org.apache.flink.runtime.operators.TempBarrier

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.