Examples of org.apache.hadoop.yarn.api.records.Container

Package org.apache.hadoop.yarn.api.records

Examples of org.apache.hadoop.yarn.api.records.Container

org.apache.hadoop.yarn.api.records.Container
Container represents an allocated resource in the cluster.

The ResourceManager is the sole authority to allocate any Container to applications. The allocated Container is always on a single node and has a unique {@link ContainerId}. It has a specific amount of {@link Resource} allocated.
It includes details such as:
- {@link ContainerId} for the container, which is globally unique.
- {@link NodeId} of the node on which it is allocated.
- HTTP uri of the node.
- {@link Resource} allocated to the container.
- {@link Priority} at which the container was allocated.
- {@link ContainerState} of the container.
- {@link ContainerToken} of the container, used to securely verify authenticity of the allocation.
- {@link ContainerStatus} of the container.
Typically, an ApplicationMaster receives the Container from the ResourceManager during resource-negotiation and then talks to the NodManager to start/stop containers.
@see AMRMProtocol#allocate(org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest) @see ContainerManager#startContainer(org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest) @see ContainerManager#stopContainer(org.apache.hadoop.yarn.api.protocolrecords.StopContainerRequest)

    }
    assert Resources.greaterThan(
        resourceCalculator, clusterResource, available, Resources.none());


    // Create the container if necessary
    Container container = 
        getContainer(rmContainer, application, node, capability, priority);
  
    // something went wrong getting/creating the container 
    if (container == null) {
      LOG.warn("Couldn't get container for allocation!");
      return Resources.none();
    }


    // Can we allocate a container on this node?
    int availableContainers = 
        resourceCalculator.computeAvailableContainers(available, capability);
    if (availableContainers > 0) {
      // Allocate...


      // Did we previously reserve containers at this 'priority'?
      if (rmContainer != null){
        unreserve(application, priority, node, rmContainer);
      }


      // Inform the application
      RMContainer allocatedContainer = 
          application.allocate(type, node, priority, request, container);


      // Does the application need this resource?
      if (allocatedContainer == null) {
        return Resources.none();
      }


      // Inform the node
      node.allocateContainer(allocatedContainer);


      LOG.info("assignedContainer" +
          " application attempt=" + application.getApplicationAttemptId() +
          " container=" + container + 
          " queue=" + this + 
          " clusterResource=" + clusterResource);


      return container.getResource();
    } else {
      // Reserve by 'charging' in advance...
      reserve(application, priority, node, rmContainer, container);


      LOG.info("Reserved container " +

View Full Code Here

  }


  synchronized public void containerCompleted(RMContainer rmContainer,
      ContainerStatus containerStatus, RMContainerEventType event) {
    
    Container container = rmContainer.getContainer();
    ContainerId containerId = container.getId();
    
    // Remove from the list of newly allocated containers if found
    newlyAllocatedContainers.remove(rmContainer);
    
    // Inform the container

View Full Code Here

      boolean removed = false;


      // Careful! Locking order is important!
      synchronized (this) {


        Container container = rmContainer.getContainer();


        // Inform the application & the node
        // Note: It's safe to assume that all state changes to RMContainer
        // happen under scheduler's lock... 
        // So, this is, in effect, a transaction across application & node
        if (rmContainer.getState() == RMContainerState.RESERVED) {
          removed = unreserve(application, rmContainer.getReservedPriority(),
              node, rmContainer);
        } else {
          removed =
            application.containerCompleted(rmContainer, containerStatus, event);
          node.releaseContainer(container);
        }


        // Book-keeping
        if (removed) {
          releaseResource(clusterResource,
              application, container.getResource());
          LOG.info("completedContainer" +
              " container=" + container +
              " queue=" + this +
              " cluster=" + clusterResource);
        }

View Full Code Here

            .isAllSuccessCallsExecuted()) {
          Thread.sleep(10);
        }
        asyncClient.setClient(mockNMClient(1));
      }
      Container container = mockContainer(i);
      ContainerLaunchContext clc =
          recordFactory.newRecordInstance(ContainerLaunchContext.class);
      asyncClient.startContainerAsync(container, clc);
    }
    while (!((TestCallbackHandler1) asyncClient.getCallbackHandler())
        .isStartAndQueryFailureCallsExecuted()) {
      Thread.sleep(10);
    }
    asyncClient.setClient(mockNMClient(2));
    ((TestCallbackHandler1) asyncClient.getCallbackHandler()).path = false;
    for (int i = 0; i < expectedFailure; ++i) {
      Container container = mockContainer(
          expectedSuccess + expectedFailure + i);
      ContainerLaunchContext clc =
          recordFactory.newRecordInstance(ContainerLaunchContext.class);
      asyncClient.startContainerAsync(container, clc);
    }

View Full Code Here

    CyclicBarrier barrierC = new CyclicBarrier(2);
    asyncClient = new MockNMClientAsync2(barrierA, barrierB, barrierC);
    asyncClient.init(new Configuration());
    asyncClient.start();


    final Container container = mockContainer(1);
    final ContainerLaunchContext clc =
        recordFactory.newRecordInstance(ContainerLaunchContext.class);


    // start container from another thread
    Thread t = new Thread() {
      @Override
      public void run() {
        asyncClient.startContainerAsync(container, clc);
      }
    };
    t.start();


    barrierA.await();
    asyncClient.stopContainerAsync(container.getId(), container.getNodeId());
    barrierC.await();


    Assert.assertFalse("Starting and stopping should be out of order",
        ((TestCallbackHandler2) asyncClient.getCallbackHandler())
            .exceptionOccurred.get());

View Full Code Here

    NodeId nodeId = node.getRMNode().getNodeID();
    ContainerId containerId = BuilderUtils.newContainerId(application
        .getApplicationAttemptId(), application.getNewContainerId());


    // Create the container
    Container container =
        BuilderUtils.newContainer(containerId, nodeId, node.getRMNode()
          .getHttpAddress(), capability, priority, null);


    return container;
  }

View Full Code Here

    Resource capability = request.getCapability();


    // How much does the node have?
    Resource available = node.getAvailableResource();


    Container container = null;
    if (reserved) {
      container = node.getReservedContainer().getContainer();
    } else {
      container = createContainer(app, node, capability, request.getPriority());
    }


    // Can we allocate a container on this node?
    if (Resources.fitsIn(capability, available)) {
      // Inform the application of the new container for this request
      RMContainer allocatedContainer =
          app.allocate(type, node, request.getPriority(), request, container);
      if (allocatedContainer == null) {
        // Did the application need this resource?
        if (reserved) {
          unreserve(request.getPriority(), node);
        }
        return Resources.none();
      }


      // If we had previously made a reservation, delete it
      if (reserved) {
        unreserve(request.getPriority(), node);
      }


      // Inform the node
      node.allocateContainer(allocatedContainer);


      // If this container is used to run AM, update the leaf queue's AM usage
      if (app.getLiveContainers().size() == 1 &&
          !app.getUnmanagedAM()) {
        queue.addAMResourceUsage(container.getResource());
        app.setAmRunning(true);
      }


      return container.getResource();
    } else {
      // The desired container won't fit here, so reserve
      reserve(request.getPriority(), node, container, reserved);


      return FairScheduler.CONTAINER_RESERVED;

View Full Code Here

    if (rmContainer == null) {
      LOG.info("Null container completed...");
      return;
    }


    Container container = rmContainer.getContainer();


    // Get the application for the finished container
    FSSchedulerApp application =
        getCurrentAttemptForContainer(container.getId());
    ApplicationId appId =
        container.getId().getApplicationAttemptId().getApplicationId();
    if (application == null) {
      LOG.info("Container " + container + " of" +
          " unknown application attempt " + appId +
          " completed with event " + event);
      return;
    }


    // Get the node on which the container was allocated
    FSSchedulerNode node = getFSSchedulerNode(container.getNodeId());


    if (rmContainer.getState() == RMContainerState.RESERVED) {
      application.unreserve(node, rmContainer.getReservedPriority());
      node.unreserveResource(application);
    } else {
      application.containerCompleted(rmContainer, containerStatus, event);
      node.releaseContainer(container);
      updateRootQueueMetrics();
    }


    LOG.info("Application attempt " + application.getApplicationAttemptId()
        + " released container " + container.getId() + " on node: " + node
        + " with event: " + event);
  }

View Full Code Here

            
      // set master container for the current running AMContainer for this
      // attempt.
      RMAppAttempt appAttempt = rmApp.getCurrentAppAttempt();
      if (appAttempt != null) {
        Container masterContainer = appAttempt.getMasterContainer();


        // Mark current running AMContainer's RMContainer based on the master
        // container ID stored in AppAttempt.
        if (masterContainer != null
            && masterContainer.getId().equals(rmContainer.getContainerId())) {
          ((RMContainerImpl)rmContainer).setAMContainer(true);
        }
      }
    }
  }

View Full Code Here

    }
  }


  private RMContainer recoverAndCreateContainer(NMContainerStatus status,
      RMNode node) {
    Container container =
        Container.newInstance(status.getContainerId(), node.getNodeID(),
          node.getHttpAddress(), status.getAllocatedResource(),
          status.getPriority(), null);
    ApplicationAttemptId attemptId =
        container.getId().getApplicationAttemptId();
    RMContainer rmContainer =
        new RMContainerImpl(container, attemptId, node.getNodeID(),
          applications.get(attemptId.getApplicationId()).getUser(), rmContext,
          status.getCreationTime());
    return rmContainer;

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.hadoop.yarn.api.records.Container

com.cloudera.iterativereduce.yarn.appmaster.ApplicationMaster

com.cloudera.kitten.appmaster.service.ApplicationMasterServiceImpl$ContainerTracker

org.apache.giraph.yarn.GiraphApplicationMaster$NMCallbackHandler

org.apache.hadoop.mapred.TestLocalContainerLauncher

org.apache.hadoop.mapreduce.v2.app.job.impl.TaskAttemptImpl$ContainerAssignedTransition

org.apache.hadoop.mapreduce.v2.app.job.impl.TestTaskAttempt

org.apache.hadoop.mapreduce.v2.app.local.LocalContainerAllocator

org.apache.hadoop.mapreduce.v2.app.MRApp

org.apache.hadoop.mapreduce.v2.app.MRApp$MRAppContainerAllocator

org.apache.hadoop.mapreduce.v2.app.MRAppBenchmark$ThrottledMRApp$ThrottledContainerAllocator

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.