Package org.apache.flink.runtime.io.network.bufferprovider

Examples of org.apache.flink.runtime.io.network.bufferprovider.GlobalBufferPool


    this.channelLookupService = channelLookupService;
    this.connectionInfo = connectionInfo;

    try {
      this.globalBufferPool = new GlobalBufferPool(numNetworkBuffers, networkBufferSize);
    } catch (Throwable e) {
      throw new IOException("Failed to instantiate GlobalBufferPool.", e);
    }

    this.networkConnectionManager = networkConnectionManager;
View Full Code Here


     
      final JobGraph jobGraph = new JobGraph("Pointwise Job", sender, receiver);
     
      final JobManager jm = startJobManager(NUM_TASKS);
     
      final GlobalBufferPool bp = ((LocalInstanceManager) jm.getInstanceManager())
          .getTaskManagers()[0].getChannelManager().getGlobalBufferPool();
     
      try {
        JobSubmissionResult result = jm.submitJob(jobGraph);

        if (result.getReturnCode() != AbstractJobResult.ReturnCode.SUCCESS) {
          System.out.println(result.getDescription());
        }
        assertEquals(AbstractJobResult.ReturnCode.SUCCESS, result.getReturnCode());
       
        // monitor the execution
        ExecutionGraph eg = jm.getCurrentJobs().get(jobGraph.getJobID());
       
        if (eg != null) {
          eg.waitForJobEnd();
          assertEquals(JobStatus.FINISHED, eg.getState());
        }
        else {
          // already done, that was fast;
        }
       
        // make sure that in any case, the network buffers are all returned
        waitForTaskThreadsToBeTerminated();
        assertEquals(bp.numBuffers(), bp.numAvailableBuffers());
      }
      finally {
        jm.shutdown();
      }
    }
View Full Code Here

      final JobManager jm = startJobManager(2, NUM_TASKS / 2);
     
      final TaskManager tm1 = ((LocalInstanceManager) jm.getInstanceManager()).getTaskManagers()[0];
      final TaskManager tm2 = ((LocalInstanceManager) jm.getInstanceManager()).getTaskManagers()[1];
     
      final GlobalBufferPool bp1 = tm1.getChannelManager().getGlobalBufferPool();
      final GlobalBufferPool bp2 = tm2.getChannelManager().getGlobalBufferPool();
     
      try {
        JobSubmissionResult result = jm.submitJob(jobGraph);

        if (result.getReturnCode() != AbstractJobResult.ReturnCode.SUCCESS) {
          System.out.println(result.getDescription());
        }
        assertEquals(AbstractJobResult.ReturnCode.SUCCESS, result.getReturnCode());
       
        ExecutionGraph eg = jm.getCurrentJobs().get(jobGraph.getJobID());
       
        // wait until everyone has settled in
        long deadline = System.currentTimeMillis() + 2000;
        while (System.currentTimeMillis() < deadline) {
         
          boolean allrunning = true;
          for (ExecutionVertex v : eg.getJobVertex(receiver.getID()).getTaskVertices()) {
            if (v.getCurrentExecutionAttempt().getState() != ExecutionState.RUNNING) {
              allrunning = false;
              break;
            }
          }
         
          if (allrunning) {
            break;
          }
          Thread.sleep(200);
        }
       
        // kill one task manager
        TaskManager tm = ((LocalInstanceManager) jm.getInstanceManager()).getTaskManagers()[0];
        tm.shutdown();
       
        eg.waitForJobEnd();
       
        // make sure that in any case, the network buffers are all returned
        waitForTaskThreadsToBeTerminated();
        assertTrue(bp1.isDestroyed() || bp1.numBuffers() == bp1.numAvailableBuffers());
        assertTrue(bp2.isDestroyed() || bp2.numBuffers() == bp2.numAvailableBuffers());
      }
      finally {
        jm.shutdown();
      }
    }
View Full Code Here

     
      final JobGraph jobGraph = new JobGraph("Test Job", vertex);
     
      final JobManager jm = startJobManager(1);
     
      final GlobalBufferPool bp = ((LocalInstanceManager) jm.getInstanceManager())
          .getTaskManagers()[0].getChannelManager().getGlobalBufferPool();
     
      try {
       
        assertEquals(1, jm.getTotalNumberOfRegisteredSlots());
       
        JobSubmissionResult result = jm.submitJob(jobGraph);
        assertEquals(AbstractJobResult.ReturnCode.ERROR, result.getReturnCode());
       
        // monitor the execution
        ExecutionGraph eg = jm.getCurrentJobs().get(jobGraph.getJobID());
       
        if (eg != null) {
         
          long deadline = System.currentTimeMillis() + 60*1000;
          boolean success = false;
         
          while (System.currentTimeMillis() < deadline) {
            JobStatus state = eg.getState();
            if (state == JobStatus.FINISHED) {
              success = true;
              break;
            }
            else if (state == JobStatus.FAILED || state == JobStatus.CANCELED) {
              break;
            }
            else {
              Thread.sleep(200);
            }
          }
         
          assertTrue("The job did not finish successfully.", success);
         
//          assertEquals(0, eg.getRegisteredExecutions().size());
        }
        else {
          // already done, that was fast;
        }
       
       
       
        // make sure that in any case, the network buffers are all returned
        waitForTaskThreadsToBeTerminated();
        assertEquals(bp.numBuffers(), bp.numAvailableBuffers());
      }
      finally {
        jm.shutdown();
      }
    }
View Full Code Here

     
      final JobGraph jobGraph = new JobGraph("Test Job", vertex);
     
      final JobManager jm = startJobManager(NUM_TASKS);
     
      final GlobalBufferPool bp = ((LocalInstanceManager) jm.getInstanceManager())
          .getTaskManagers()[0].getChannelManager().getGlobalBufferPool();
     
      try {
       
        assertEquals(NUM_TASKS, jm.getTotalNumberOfRegisteredSlots());
       
        JobSubmissionResult result = jm.submitJob(jobGraph);
       
        if (result.getReturnCode() != AbstractJobResult.ReturnCode.SUCCESS) {
          System.out.println(result.getDescription());
        }
        assertEquals(AbstractJobResult.ReturnCode.SUCCESS, result.getReturnCode());
       
        // monitor the execution
        ExecutionGraph eg = jm.getCurrentJobs().get(jobGraph.getJobID());
       
        if (eg != null) {
         
          long deadline = System.currentTimeMillis() + 60*1000;
          boolean success = false;
         
          while (System.currentTimeMillis() < deadline) {
            JobStatus state = eg.getState();
            if (state == JobStatus.FINISHED) {
              success = true;
              break;
            }
            else if (state == JobStatus.FAILED || state == JobStatus.CANCELED) {
              break;
            }
            else {
              Thread.sleep(200);
            }
          }
         
          assertTrue("The job did not finish successfully.", success);
         
//          assertEquals(0, eg.getRegisteredExecutions().size());
        }
        else {
          // already done, that was fast;
        }
       
        // make sure that in any case, the network buffers are all returned
        waitForTaskThreadsToBeTerminated();
        assertEquals(bp.numBuffers(), bp.numAvailableBuffers());
      }
      finally {
        jm.shutdown();
      }
    }
View Full Code Here

      final JobGraph jobGraph = new JobGraph("Test Job", vertex);
      jobGraph.setAllowQueuedScheduling(true);
     
      final JobManager jm = startJobManager(10);
     
      final GlobalBufferPool bp = ((LocalInstanceManager) jm.getInstanceManager())
          .getTaskManagers()[0].getChannelManager().getGlobalBufferPool();
     
      try {
        JobSubmissionResult result = jm.submitJob(jobGraph);
       
        if (result.getReturnCode() != AbstractJobResult.ReturnCode.SUCCESS) {
          System.out.println(result.getDescription());
        }
        assertEquals(AbstractJobResult.ReturnCode.SUCCESS, result.getReturnCode());
       
        // monitor the execution
        ExecutionGraph eg = jm.getCurrentJobs().get(jobGraph.getJobID());
       
        if (eg != null) {
          eg.waitForJobEnd();
          assertEquals(JobStatus.FINISHED, eg.getState());
         
//          assertEquals(0, eg.getRegisteredExecutions().size());
        }
        else {
          // already done, that was fast;
        }
       
        // make sure that in any case, the network buffers are all returned
        waitForTaskThreadsToBeTerminated();
        assertEquals(bp.numBuffers(), bp.numAvailableBuffers());
      }
      finally {
        jm.shutdown();
      }
    }
View Full Code Here

     
      final JobGraph jobGraph = new JobGraph("Pointwise Job", sender, receiver);
     
      final JobManager jm = startJobManager(2 * NUM_TASKS);
     
      final GlobalBufferPool bp = ((LocalInstanceManager) jm.getInstanceManager())
          .getTaskManagers()[0].getChannelManager().getGlobalBufferPool();
     
      try {
        JobSubmissionResult result = jm.submitJob(jobGraph);

        if (result.getReturnCode() != AbstractJobResult.ReturnCode.SUCCESS) {
          System.out.println(result.getDescription());
        }
        assertEquals(AbstractJobResult.ReturnCode.SUCCESS, result.getReturnCode());
       
        // monitor the execution
        ExecutionGraph eg = jm.getCurrentJobs().get(jobGraph.getJobID());
       
        if (eg != null) {
          eg.waitForJobEnd();
          assertEquals(JobStatus.FINISHED, eg.getState());
         
//          assertEquals(0, eg.getRegisteredExecutions().size());
        }
        else {
          // already done, that was fast;
        }
       
        // make sure that in any case, the network buffers are all returned
        waitForTaskThreadsToBeTerminated();
        assertEquals(bp.numBuffers(), bp.numAvailableBuffers());
      }
      finally {
        jm.shutdown();
      }
    }
View Full Code Here

     
      final JobGraph jobGraph = new JobGraph("Bipartite Job", sender, receiver);
     
      final JobManager jm = startJobManager(2 * NUM_TASKS);
     
      final GlobalBufferPool bp = ((LocalInstanceManager) jm.getInstanceManager())
          .getTaskManagers()[0].getChannelManager().getGlobalBufferPool();
     
      try {
        JobSubmissionResult result = jm.submitJob(jobGraph);

        if (result.getReturnCode() != AbstractJobResult.ReturnCode.SUCCESS) {
          System.out.println(result.getDescription());
        }
        assertEquals(AbstractJobResult.ReturnCode.SUCCESS, result.getReturnCode());
       
        // monitor the execution
        ExecutionGraph eg = jm.getCurrentJobs().get(jobGraph.getJobID());
       
        if (eg != null) {
          eg.waitForJobEnd();
          assertEquals(JobStatus.FINISHED, eg.getState());
         
//          assertEquals(0, eg.getRegisteredExecutions().size());
        }
        else {
          // already done, that was fast;
        }
       
        // make sure that in any case, the network buffers are all returned
        waitForTaskThreadsToBeTerminated();
        assertEquals(bp.numBuffers(), bp.numAvailableBuffers());
      }
      finally {
        jm.shutdown();
      }
    }
View Full Code Here

     
      final JobGraph jobGraph = new JobGraph("Bipartite Job", sender1, receiver, sender2);
     
      final JobManager jm = startJobManager(6*NUM_TASKS);
     
      final GlobalBufferPool bp = ((LocalInstanceManager) jm.getInstanceManager())
          .getTaskManagers()[0].getChannelManager().getGlobalBufferPool();
     
      try {
        JobSubmissionResult result = jm.submitJob(jobGraph);

        if (result.getReturnCode() != AbstractJobResult.ReturnCode.SUCCESS) {
          System.out.println(result.getDescription());
        }
        assertEquals(AbstractJobResult.ReturnCode.SUCCESS, result.getReturnCode());
       
        // monitor the execution
        ExecutionGraph eg = jm.getCurrentJobs().get(jobGraph.getJobID());
       
        if (eg != null) {
          eg.waitForJobEnd();
          assertEquals(JobStatus.FAILED, eg.getState());
         
//          assertEquals(0, eg.getRegisteredExecutions().size());
        }
        else {
          // already done, that was fast;
        }
       
        // make sure that in any case, the network buffers are all returned
        waitForTaskThreadsToBeTerminated();
        assertEquals(bp.numBuffers(), bp.numAvailableBuffers());
      }
      finally {
        jm.shutdown();
      }
    }
View Full Code Here

     
      final JobGraph jobGraph = new JobGraph("Bipartite Job", sender1, receiver, sender2);
     
      JobManager jm = startJobManager(6 * NUM_TASKS);
     
      final GlobalBufferPool bp = ((LocalInstanceManager) jm.getInstanceManager())
                .getTaskManagers()[0].getChannelManager().getGlobalBufferPool();
     
      try {
        JobSubmissionResult result = jm.submitJob(jobGraph);

        if (result.getReturnCode() != AbstractJobResult.ReturnCode.SUCCESS) {
          System.out.println(result.getDescription());
        }
        assertEquals(AbstractJobResult.ReturnCode.SUCCESS, result.getReturnCode());
       
        // monitor the execution
        ExecutionGraph eg = jm.getCurrentJobs().get(jobGraph.getJobID());
       
        if (eg != null) {
          eg.waitForJobEnd();
          assertEquals(JobStatus.FINISHED, eg.getState());
         
//          assertEquals(0, eg.getRegisteredExecutions().size());
        }
        else {
          // already done, that was fast;
        }
       
        // make sure that in any case, the network buffers are all returned
        waitForTaskThreadsToBeTerminated();
        assertEquals(bp.numBuffers(), bp.numAvailableBuffers());
      }
      finally {
        jm.shutdown();
      }
    }
View Full Code Here

TOP

Related Classes of org.apache.flink.runtime.io.network.bufferprovider.GlobalBufferPool

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.