Examples of org.apache.hadoop.yarn.api.records.Container

Package org.apache.hadoop.yarn.api.records

Examples of org.apache.hadoop.yarn.api.records.Container

org.apache.hadoop.yarn.api.records.Container
Container represents an allocated resource in the cluster.

The ResourceManager is the sole authority to allocate any Container to applications. The allocated Container is always on a single node and has a unique {@link ContainerId}. It has a specific amount of {@link Resource} allocated.
It includes details such as:
- {@link ContainerId} for the container, which is globally unique.
- {@link NodeId} of the node on which it is allocated.
- HTTP uri of the node.
- {@link Resource} allocated to the container.
- {@link Priority} at which the container was allocated.
- {@link ContainerState} of the container.
- {@link ContainerToken} of the container, used to securely verify authenticity of the allocation.
- {@link ContainerStatus} of the container.
Typically, an ApplicationMaster receives the Container from the ResourceManager during resource-negotiation and then talks to the NodManager to start/stop containers.
@see AMRMProtocol#allocate(org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest) @see ContainerManager#startContainer(org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest) @see ContainerManager#stopContainer(org.apache.hadoop.yarn.api.protocolrecords.StopContainerRequest)

        new ArrayList<Container>(newlyAllocatedContainers.size());
    List<NMToken> nmTokens = new ArrayList<NMToken>();
    for (Iterator<RMContainer> i = newlyAllocatedContainers.iterator(); i
      .hasNext();) {
      RMContainer rmContainer = i.next();
      Container container = rmContainer.getContainer();
      try {
        // create container token and NMToken altogether.
        container.setContainerToken(rmContext.getContainerTokenSecretManager()
          .createContainerToken(container.getId(), container.getNodeId(),
            getUser(), container.getResource(), container.getPriority(),
            rmContainer.getCreationTime()));
        NMToken nmToken =
            rmContext.getNMTokenSecretManager().createAndGetNMToken(getUser(),
              getApplicationAttemptId(), container);
        if (nmToken != null) {
          nmTokens.add(nmToken);
        }
      } catch (IllegalArgumentException e) {
        // DNS might be down, skip returning this container.
        LOG.error("Error trying to assign container token and NM token to" +
            " an allocated container " + container.getId(), e);
        continue;
      }
      returnContainerList.add(container);
      i.remove();
      rmContainer.handle(new RMContainerEvent(rmContainer.getContainerId(),

View Full Code Here

    }
    
    // Remove from the list of newly allocated containers if found
    newlyAllocatedContainers.remove(rmContainer);


    Container container = rmContainer.getContainer();
    ContainerId containerId = container.getId();


    // Inform the container
    rmContainer.handle(
        new RMContainerFinishedEvent(
            containerId,

View Full Code Here

        NodeId nodeId = node.getRMNode().getNodeID();
        ContainerId containerId = BuilderUtils.newContainerId(application
            .getApplicationAttemptId(), application.getNewContainerId());


        // Create the container
        Container container =
            BuilderUtils.newContainer(containerId, nodeId, node.getRMNode()
              .getHttpAddress(), capability, priority, null);
        
        // Allocate!

View Full Code Here

      LOG.info("Null container completed...");
      return;
    }


    // Get the application for the finished container
    Container container = rmContainer.getContainer();
    FiCaSchedulerApp application =
        getCurrentAttemptForContainer(container.getId());
    ApplicationId appId =
        container.getId().getApplicationAttemptId().getApplicationId();
    
    // Get the node on which the container was allocated
    FiCaSchedulerNode node = getNode(container.getNodeId());
    
    if (application == null) {
      LOG.info("Unknown application: " + appId + 
          " released container " + container.getId() +
          " on node: " + node + 
          " with event: " + event);
      return;
    }


    // Inform the application
    application.containerCompleted(rmContainer, containerStatus, event);


    // Inform the node
    node.releaseContainer(container);
    
    // Update total usage
    Resources.subtractFrom(usedResource, container.getResource());


    LOG.info("Application attempt " + application.getApplicationAttemptId() + 
        " released container " + container.getId() +
        " on node: " + node + 
        " with event: " + event);
     
  }

View Full Code Here

   * 
   * @param rmContainer
   *          allocated container
   */
  public synchronized void allocateContainer(RMContainer rmContainer) {
    Container container = rmContainer.getContainer();
    deductAvailableResource(container.getResource());
    ++numContainers;


    launchedContainers.put(container.getId(), rmContainer);


    LOG.info("Assigned container " + container.getId() + " of capacity "
        + container.getResource() + " on host " + rmNode.getNodeAddress()
        + ", which has " + numContainers + " containers, "
        + getUsedResource() + " used and " + getAvailableResource()
        + " available after allocation");
  }

View Full Code Here

        rm.getResourceScheduler().getRootQueueMetrics()
            .getAvailableVirtualCores());


    // Allocate one container on node1
    ContainerId cId1 = newContainerId(1, 1, 1);
    Container container1 = Container.newInstance(cId1, null, null,
        Resources.createResource(GB, 1), null, null);
    node1.addNewContainer(container1, 100000l);
    Assert.assertTrue("Node1 should have one running container.",
        node1.getRunningContainers().containsKey(cId1));


    // Allocate one AM container on node1
    ContainerId cId2 = newContainerId(2, 1, 1);
    Container container2 = Container.newInstance(cId2, null, null,
        Resources.createResource(GB, 1), null, null);
    node1.addNewContainer(container2, -1l);
    Assert.assertTrue("Node1 should have one running AM container",
        node1.getAMContainers().contains(cId2));

View Full Code Here

      Resources.subtractFrom(pendingResource, container.getResource());
    }
    // container released from AM
    SchedulerAppReport report = scheduler.getSchedulerAppInfo(attemptId);
    for (ContainerId containerId : containerIds) {
      Container container = null;
      for (RMContainer c : report.getLiveContainers()) {
        if (c.getContainerId().equals(containerId)) {
          container = c.getContainer();
          break;
        }
      }
      if (container != null) {
        // released allocated containers
        Resources.subtractFrom(allocatedResource, container.getResource());
      } else {
        for (RMContainer c : report.getReservedContainers()) {
          if (c.getContainerId().equals(containerId)) {
            container = c.getContainer();
            break;
          }
        }
        if (container != null) {
          // released reserved containers
          Resources.subtractFrom(pendingResource, container.getResource());
        }
      }
    }
    // containers released/preemption from scheduler
    Set<ContainerId> preemptionContainers = new HashSet<ContainerId>();
    if (allocation.getContainerPreemptions() != null) {
      preemptionContainers.addAll(allocation.getContainerPreemptions());
    }
    if (allocation.getStrictContainerPreemptions() != null) {
      preemptionContainers.addAll(allocation.getStrictContainerPreemptions());
    }
    if (! preemptionContainers.isEmpty()) {
      for (ContainerId containerId : preemptionContainers) {
        if (! preemptionContainerMap.containsKey(containerId)) {
          Container container = null;
          for (RMContainer c : report.getLiveContainers()) {
            if (c.getContainerId().equals(containerId)) {
              container = c.getContainer();
              break;
            }
          }
          if (container != null) {
            preemptionContainerMap.put(containerId, container.getResource());
          }
        }


      }
    }

View Full Code Here


    // waiting until the AM container is allocated
    while (true) {
      if (response != null && ! response.getAllocatedContainers().isEmpty()) {
        // get AM container
        Container container = response.getAllocatedContainers().get(0);
        se.getNmMap().get(container.getNodeId())
                .addNewContainer(container, -1L);
        // start AM container
        amContainer = container;
        LOG.debug(MessageFormat.format("Application {0} starts its " +
                "AM container ({1}).", appId, amContainer.getId()));

View Full Code Here

            final FiCaSchedulerApp application = 
                (FiCaSchedulerApp)(invocation.getArguments()[0]);
            final ContainerId containerId =                 
                TestUtils.getMockContainerId(application);


            Container container = TestUtils.getMockContainer(
                containerId,
                ((FiCaSchedulerNode)(invocation.getArguments()[1])).getNodeID(), 
                (Resource)(invocation.getArguments()[2]),
                ((Priority)invocation.getArguments()[3]));
            return container;

View Full Code Here

    TaskAttemptId get(ContainerId cId) {
      return containerToAttemptMap.get(cId);
    }


    ContainerId get(TaskAttemptId tId) {
      Container taskContainer;
      if (tId.getTaskId().getTaskType().equals(TaskType.MAP)) {
        taskContainer = maps.get(tId);
      } else {
        taskContainer = reduces.get(tId);
      }


      if (taskContainer == null) {
        return null;
      } else {
        return taskContainer.getId();
      }
    }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.hadoop.yarn.api.records.Container

com.cloudera.iterativereduce.yarn.appmaster.ApplicationMaster

com.cloudera.kitten.appmaster.service.ApplicationMasterServiceImpl$ContainerTracker

org.apache.giraph.yarn.GiraphApplicationMaster$NMCallbackHandler

org.apache.hadoop.mapred.TestLocalContainerLauncher

org.apache.hadoop.mapreduce.v2.app.job.impl.TaskAttemptImpl$ContainerAssignedTransition

org.apache.hadoop.mapreduce.v2.app.job.impl.TestTaskAttempt

org.apache.hadoop.mapreduce.v2.app.local.LocalContainerAllocator

org.apache.hadoop.mapreduce.v2.app.MRApp

org.apache.hadoop.mapreduce.v2.app.MRApp$MRAppContainerAllocator

org.apache.hadoop.mapreduce.v2.app.MRAppBenchmark$ThrottledMRApp$ThrottledContainerAllocator

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.