Package org.apache.flink.runtime.memorymanager

Examples of org.apache.flink.runtime.memorymanager.MemoryManager


  @Override
  public void prepare() throws Exception{
    final TaskConfig config = this.taskContext.getTaskConfig();
   
    // obtain task manager's memory manager and I/O manager
    final MemoryManager memoryManager = this.taskContext.getMemoryManager();
    final IOManager ioManager = this.taskContext.getIOManager();
   
    // set up memory and I/O parameters
    final double fractionAvailableMemory = config.getRelativeMemoryDriver();
    final int numPages = memoryManager.computeNumberOfPages(fractionAvailableMemory);
   
    // test minimum memory requirements
    final DriverStrategy ls = config.getDriverStrategy();
   
    final MutableObjectIterator<IT1> in1 = this.taskContext.getInput(0);
View Full Code Here


    // 2nd, shutdown I/O
    this.ioManager.shutdown();
    Assert.assertTrue("I/O Manager has not properly shut down.", this.ioManager.isProperlyShutDown());

    // last, verify all memory is returned and shutdown mem manager
    MemoryManager memMan = getMemoryManager();
    if (memMan != null) {
      Assert.assertTrue("Memory Manager managed memory was not completely freed.", memMan.verifyEmpty());
      memMan.shutdown();
    }
  }
View Full Code Here

  }

  @After
  public void shutdownMemoryManager() throws Exception {
    if (this.memorySize > 0) {
      MemoryManager memMan = getMemoryManager();
      if (memMan != null) {
        Assert.assertTrue("Memory Manager managed memory was not completely freed.", memMan.verifyEmpty());
        memMan.shutdown();
      }
    }
  }
View Full Code Here

   *
   * @param numInputs
   */
  protected void initLocalStrategies(int numInputs) throws Exception {

    final MemoryManager memMan = getMemoryManager();
    final IOManager ioMan = getIOManager();

    this.localStrategies = new CloseableInputProvider[numInputs];
    this.inputs = new MutableObjectIterator[numInputs];
    this.excludeFromReset = new boolean[numInputs];
    this.inputIsCached = new boolean[numInputs];
    this.inputIsAsyncMaterialized = new boolean[numInputs];
    this.materializationMemory = new int[numInputs];

    // set up the local strategies first, such that the can work before any temp barrier is created
    for (int i = 0; i < numInputs; i++) {
      initInputLocalStrategy(i);
    }

    // we do another loop over the inputs, because we want to instantiate all
    // sorters, etc before requesting the first input (as this call may block)

    // we have two types of materialized inputs, and both are replayable (can act as a cache)
    // The first variant materializes in a different thread and hence
    // acts as a pipeline breaker. this one should only be there, if a pipeline breaker is needed.
    // the second variant spills to the side and will not read unless the result is also consumed
    // in a pipelined fashion.
    this.resettableInputs = new SpillingResettableMutableObjectIterator[numInputs];
    this.tempBarriers = new TempBarrier[numInputs];

    for (int i = 0; i < numInputs; i++) {
      final int memoryPages;
      final boolean async = this.config.isInputAsynchronouslyMaterialized(i);
      final boolean cached =  this.config.isInputCached(i);

      this.inputIsAsyncMaterialized[i] = async;
      this.inputIsCached[i] = cached;

      if (async || cached) {
        memoryPages = memMan.computeNumberOfPages(this.config.getRelativeInputMaterializationMemory(i));
        if (memoryPages <= 0) {
          throw new Exception("Input marked as materialized/cached, but no memory for materialization provided.");
        }
        this.materializationMemory[i] = memoryPages;
      } else {
View Full Code Here

        this.localStrategies[i].close();
        this.localStrategies[i] = null;
      }
    }

    final MemoryManager memMan = getMemoryManager();
    final IOManager ioMan = getIOManager();

    // reset the caches, or re-run the input local strategy
    for (int i = 0; i < this.inputs.length; i++) {
      if (this.excludeFromReset[i]) {
View Full Code Here

      UnilateralSortMerger<String> sorter = null;
      BufferedReader reader = null;
      BufferedReader verifyReader = null;
     
      try {
        MemoryManager mm = new DefaultMemoryManager(1024 * 1024, 1);
        IOManager ioMan = new IOManager();
         
        TypeSerializer<String> serializer = StringSerializer.INSTANCE;
        TypeComparator<String> comparator = new StringComparator(true);
       
View Full Code Here

      UnilateralSortMerger<Tuple2<String, String[]>> sorter = null;
      BufferedReader reader = null;
      BufferedReader verifyReader = null;
     
      try {
        MemoryManager mm = new DefaultMemoryManager(1024 * 1024, 1);
        IOManager ioMan = new IOManager();
         
        TupleTypeInfo<Tuple2<String, String[]>> typeInfo = (TupleTypeInfo<Tuple2<String, String[]>>) (TupleTypeInfo<?>) TypeInfoParser.parse("Tuple2<String, String[]>");

        TypeSerializer<Tuple2<String, String[]>> serializer = typeInfo.createSerializer();
View Full Code Here

TOP

Related Classes of org.apache.flink.runtime.memorymanager.MemoryManager

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.