Examples of TargetProviderResponse


Examples of org.apache.helix.controller.provisioner.TargetProviderResponse

  }

  @Override
  public TargetProviderResponse evaluateExistingContainers(Cluster cluster, ResourceId resourceId,
      Collection<Participant> participants) {
    TargetProviderResponse response = new TargetProviderResponse();
    // ask for two containers at a time
    List<ContainerSpec> containersToAcquire = Lists.newArrayList();
    List<Participant> containersToStart = Lists.newArrayList();
    List<Participant> containersToRelease = Lists.newArrayList();
    List<Participant> containersToStop = Lists.newArrayList();
    YarnProvisionerConfig provisionerConfig =
        (YarnProvisionerConfig) cluster.getConfig().getResourceMap().get(resourceId)
            .getProvisionerConfig();
    int targetNumContainers = provisionerConfig.getNumContainers();

    // Any container that is in a state should be put in this set
    Set<ParticipantId> existingContainersIdSet = new HashSet<ParticipantId>();

    // Cache halted containers to determine which to restart and which to release
    Map<ParticipantId, Participant> excessHaltedContainers = Maps.newHashMap();

    // Cache participants to ensure that excess participants are stopped
    Map<ParticipantId, Participant> excessActiveContainers = Maps.newHashMap();

    for (Participant participant : participants) {
      ContainerConfig containerConfig = participant.getContainerConfig();
      if (containerConfig != null && containerConfig.getState() != null) {
        ContainerState state = containerConfig.getState();
        switch (state) {
        case ACQUIRING:
          existingContainersIdSet.add(participant.getId());
          break;
        case ACQUIRED:
          // acquired containers are ready to start
          existingContainersIdSet.add(participant.getId());
          containersToStart.add(participant);
          break;
        case CONNECTING:
          existingContainersIdSet.add(participant.getId());
          break;
        case CONNECTED:
          // active containers can be stopped or kept active
          existingContainersIdSet.add(participant.getId());
          excessActiveContainers.put(participant.getId(), participant);
          break;
        case DISCONNECTED:
          // disconnected containers must be stopped
          existingContainersIdSet.add(participant.getId());
          containersToStop.add(participant);
        case HALTING:
          existingContainersIdSet.add(participant.getId());
          break;
        case HALTED:
          // halted containers can be released or restarted
          existingContainersIdSet.add(participant.getId());
          excessHaltedContainers.put(participant.getId(), participant);
          break;
        case FINALIZING:
          existingContainersIdSet.add(participant.getId());
          break;
        case FINALIZED:
          break;
        case FAILED:
          // remove the failed instance
          // _helixManager.getClusterManagmentTool().dropInstance(cluster.getId().toString(),
          // new InstanceConfig(participant.getId()));
          excessHaltedContainers.put(participant.getId(), participant);
          break;
        default:
          break;
        }
      }
    }

    for (int i = 0; i < targetNumContainers; i++) {
      ParticipantId participantId = ParticipantId.from(resourceId + "_container_" + (i));
      excessActiveContainers.remove(participantId); // don't stop this container if active
      if (excessHaltedContainers.containsKey(participantId)) {
        // Halted containers can be restarted if necessary
        // Participant participant = excessHaltedContainers.get(participantId);
        // containersToStart.add(participant);
        // excessHaltedContainers.remove(participantId); // don't release this container
      } else if (!existingContainersIdSet.contains(participantId)) {
        // Unallocated containers must be allocated
        ContainerSpec containerSpec = new ContainerSpec(participantId);
        int mem = 4096;
        if (_resourceConfig.getUserConfig() != null) {
          mem = _resourceConfig.getUserConfig().getIntField("memory", mem);
        }
        containerSpec.setMemory(mem);
        containersToAcquire.add(containerSpec);
      }
    }

    // Add all the containers that should be stopped because they fall outside the target range
    containersToStop.addAll(excessActiveContainers.values());

    // Add halted containers that should not be restarted
    containersToRelease.addAll(excessHaltedContainers.values());

    response.setContainersToAcquire(containersToAcquire);
    response.setContainersToStart(containersToStart);
    response.setContainersToRelease(containersToRelease);
    response.setContainersToStop(containersToStop);
    LOG.info("target provider response containers to acquire:" + response.getContainersToAcquire());
    LOG.info("target provider response containers to start:" + response.getContainersToStart());
    LOG.info("target provider response containers to stop:" + response.getContainersToStop());
    LOG.info("target provider response containers to release:" + response.getContainersToRelease());
    return response;
  }
View Full Code Here

Examples of org.apache.helix.controller.provisioner.TargetProviderResponse

        // The target provider is given the state of container and asked for its new state. For each
        // state there is a corresponding handler function.

        // TargetProvider should be stateless, given the state of cluster and existing participants
        // it should return the same result
        final TargetProviderResponse response =
            targetProvider.evaluateExistingContainers(cluster, resourceId, participants);

        // allocate new containers
        for (final ContainerSpec spec : response.getContainersToAcquire()) {
          final ParticipantId participantId = spec.getParticipantId();
          if (!cluster.getParticipantMap().containsKey(participantId)) {
            // create a new Participant, attach the container spec
            InstanceConfig instanceConfig = new InstanceConfig(participantId);
            instanceConfig.setInstanceEnabled(false);
            instanceConfig.setContainerSpec(spec);
            // create a helix_participant in ACQUIRING state
            instanceConfig.setContainerState(ContainerState.ACQUIRING);
            // create the helix participant and add it to cluster
            helixAdmin.addInstance(cluster.getId().toString(), instanceConfig);
            cache.requireFullRefresh();
          }
          LOG.info("Allocating container for " + participantId);
          ListenableFuture<ContainerId> future = containerProvider.allocateContainer(spec);
          FutureCallback<ContainerId> callback = new FutureCallback<ContainerId>() {
            @Override
            public void onSuccess(ContainerId containerId) {
              LOG.info("Container " + containerId + " acquired. Marking " + participantId);
              updateContainerState(cache, accessor, keyBuilder, cluster, containerId,
                  participantId, ContainerState.ACQUIRED);
            }

            @Override
            public void onFailure(Throwable t) {
              LOG.error("Could not allocate a container for participant " + participantId, t);
              updateContainerState(cache, accessor, keyBuilder, cluster, null, participantId,
                  ContainerState.FAILED);
            }
          };
          safeAddCallback(future, callback);
        }

        // start new containers
        for (final Participant participant : response.getContainersToStart()) {
          final ContainerId containerId = participant.getInstanceConfig().getContainerId();
          updateContainerState(cache, accessor, keyBuilder, cluster, null, participant.getId(),
              ContainerState.CONNECTING);
          // create the helix participant and add it to cluster
          LOG.info("Starting container " + containerId + " for " + participant.getId());
          ListenableFuture<Boolean> future =
              containerProvider.startContainer(containerId, participant);
          FutureCallback<Boolean> callback = new FutureCallback<Boolean>() {
            @Override
            public void onSuccess(Boolean result) {
              // Do nothing yet, need to wait for live instance
              LOG.info("Container " + containerId + " started for " + participant.getId());
            }

            @Override
            public void onFailure(Throwable t) {
              LOG.error("Could not start container" + containerId + "for participant "
                  + participant.getId(), t);
              updateContainerState(cache, accessor, keyBuilder, cluster, null, participant.getId(),
                  ContainerState.FAILED);
            }
          };
          safeAddCallback(future, callback);
        }

        // release containers
        for (final Participant participant : response.getContainersToRelease()) {
          // mark it as finalizing
          final ContainerId containerId = participant.getInstanceConfig().getContainerId();
          updateContainerState(cache, accessor, keyBuilder, cluster, null, participant.getId(),
              ContainerState.FINALIZING);
          // remove the participant
          LOG.info("Deallocating container " + containerId + " for " + participant.getId());
          ListenableFuture<Boolean> future = containerProvider.deallocateContainer(containerId);
          FutureCallback<Boolean> callback = new FutureCallback<Boolean>() {
            @Override
            public void onSuccess(Boolean result) {
              LOG.info("Container " + containerId + " deallocated. Dropping " + participant.getId());
              InstanceConfig existingInstance =
                  helixAdmin.getInstanceConfig(cluster.getId().toString(), participant.getId()
                      .toString());
              helixAdmin.dropInstance(cluster.getId().toString(), existingInstance);
              cache.requireFullRefresh();
            }

            @Override
            public void onFailure(Throwable t) {
              LOG.error("Could not deallocate container" + containerId + "for participant "
                  + participant.getId(), t);
              updateContainerState(cache, accessor, keyBuilder, cluster, null, participant.getId(),
                  ContainerState.FAILED);
            }
          };
          safeAddCallback(future, callback);
        }

        // stop but don't remove
        for (final Participant participant : response.getContainersToStop()) {
          // switch to halting
          final ContainerId containerId = participant.getInstanceConfig().getContainerId();
          updateContainerState(cache, accessor, keyBuilder, cluster, null, participant.getId(),
              ContainerState.HALTING);
          // stop the container
View Full Code Here

Examples of org.apache.helix.controller.provisioner.TargetProviderResponse

    }

    @Override
    public TargetProviderResponse evaluateExistingContainers(Cluster cluster,
        ResourceId resourceId, Collection<Participant> participants) {
      TargetProviderResponse response = new TargetProviderResponse();
      // ask for one container at a time
      List<ContainerSpec> containersToAcquire = Lists.newArrayList();
      boolean asked = false;
      if (_askCount < MAX_PARTICIPANTS) {
        containersToAcquire.add(new ContainerSpec(ParticipantId.from("container" + _askCount)));
        asked = true;
      }
      List<Participant> containersToStart = Lists.newArrayList();
      List<Participant> containersToStop = Lists.newArrayList();
      List<Participant> containersToRelease = Lists.newArrayList();
      int stopCount = 0;
      for (Participant participant : participants) {
        ContainerConfig containerConfig = participant.getContainerConfig();
        if (containerConfig != null && containerConfig.getState() != null) {
          ContainerState state = containerConfig.getState();
          switch (state) {
          case ACQUIRED:
            // acquired containers are ready to start
            containersToStart.add(participant);
            break;
          case CONNECTED:
            // stop at most one active at a time, wait for everything to be up first
            if (stopCount < 1 && _askCount >= MAX_PARTICIPANTS) {
              containersToStop.add(participant);
              stopCount++;
            }
            break;
          case HALTED:
            // halted containers can be released
            containersToRelease.add(participant);
            break;
          default:
            break;
          }
          ContainerId containerId = containerConfig.getId();
          if (containerId != null) {
            _containerParticipants.put(containerId, participant.getId());
            _states.put(containerId, state);
          }
        }
      }
      // update acquire request count
      if (asked) {
        _askCount++;
      }
      // set the response
      response.setContainersToAcquire(containersToAcquire);
      response.setContainersToStart(containersToStart);
      response.setContainersToStop(containersToStop);
      response.setContainersToRelease(containersToRelease);
      return response;
    }
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.