Examples of InputHost

org.apache.tez.runtime.library.shuffle.common.InputHost
Represents a Host with respect to the MapReduce ShuffleHandler. srcPhysicalIndex / partition is part of this since that only knows how to serve ine partition at a time.

Examples of org.apache.tez.runtime.library.shuffle.common.InputHost

  /////////////////// Methods for InputEventHandler
  
  public void addKnownInput(String hostName, int port,
      InputAttemptIdentifier srcAttemptIdentifier, int srcPhysicalIndex) {
    String identifier = InputHost.createIdentifier(hostName, port);
    InputHost host = knownSrcHosts.get(identifier);
    if (host == null) {
      host = new InputHost(hostName, port, inputContext.getApplicationId(), srcPhysicalIndex);
      assert identifier.equals(host.getIdentifier());
      InputHost old = knownSrcHosts.putIfAbsent(identifier, host);
      if (old != null) {
        host = old;
      }
    }
    if (LOG.isDebugEnabled()) {

View Full Code Here

Examples of org.apache.tez.runtime.library.shuffle.common.InputHost

          lock.lock();
          try {
            int maxFetchersToRun = numFetchers - runningFetchers.size();
            int count = 0;
            while (pendingHosts.peek() != null && !isShutdown.get()) {
              InputHost inputHost = null;
              try {
                inputHost = pendingHosts.take();
              } catch (InterruptedException e) {
                if (isShutdown.get()) {
                  LOG.info("Interrupted and hasBeenShutdown, Breaking out of ShuffleScheduler Loop");
                  break;
                } else {
                  throw e;
                }
              }
              if (LOG.isDebugEnabled()) {
                LOG.debug("Processing pending host: " + inputHost.toDetailedString());
              }
              if (inputHost.getNumPendingInputs() > 0 && !isShutdown.get()) {
                LOG.info("Scheduling fetch for inputHost: " + inputHost.getIdentifier());
                Fetcher fetcher = constructFetcherForHost(inputHost, conf);
                runningFetchers.add(fetcher);
                if (isShutdown.get()) {
                  LOG.info("hasBeenShutdown, Breaking out of ShuffleScheduler Loop");
                }
                ListenableFuture<FetchResult> future = fetcherExecutor
                    .submit(fetcher);
                Futures.addCallback(future, new FetchFutureCallback(fetcher));
                if (++count >= maxFetchersToRun) {
                  break;
                }
              } else {
                if (LOG.isDebugEnabled()) {
                  LOG.debug("Skipping host: " + inputHost.getIdentifier()
                      + " since it has no inputs to process");
                }
              }
            }
          } finally {

View Full Code Here

Examples of org.apache.tez.runtime.library.shuffle.common.InputHost

      if (isShutdown.get()) {
        LOG.info("Already shutdown. Ignoring event from fetcher");
      } else {
        Iterable<InputAttemptIdentifier> pendingInputs = result.getPendingInputs();
        if (pendingInputs != null && pendingInputs.iterator().hasNext()) {
          InputHost inputHost = knownSrcHosts.get(InputHost.createIdentifier(result.getHost(), result.getPort()));
          assert inputHost != null;
          for (InputAttemptIdentifier input : pendingInputs) {
            inputHost.addKnownInput(input);
          }
          pendingHosts.add(inputHost);
        }
        doBookKeepingForFetcherComplete();
      }

View Full Code Here

Examples of org.apache.tez.runtime.library.shuffle.common.InputHost

    
    @Override
    public void onSuccess(FetchResult result) {
      Iterable<InputAttemptIdentifier> pendingInputs = result.getPendingInputs();
      if (pendingInputs != null && pendingInputs.iterator().hasNext()) {
        InputHost inputHost = knownSrcHosts.get(result.getHost());
        assert inputHost != null;
        for (InputAttemptIdentifier input : pendingInputs) {
          inputHost.addKnownInput(input);
        }
        pendingHosts.add(inputHost);
      }
      doBookKeepingForFetcherComplete();
    }

View Full Code Here

Examples of org.apache.tez.runtime.library.shuffle.common.InputHost

  
  /////////////////// Methods for InputEventHandler
  
  public void addKnownInput(String hostName, int port,
      InputAttemptIdentifier srcAttemptIdentifier, int partition) {
    InputHost host = knownSrcHosts.get(hostName);
    if (host == null) {
      host = new InputHost(hostName, port, inputContext.getApplicationId());
      InputHost old = knownSrcHosts.putIfAbsent(hostName, host);
      if (old != null) {
        host = old;
      }
    }
    if (LOG.isDebugEnabled()) {

View Full Code Here

Examples of org.apache.tez.runtime.library.shuffle.common.InputHost

          lock.lock();
          try {
            int maxFetchersToRun = numFetchers - numRunningFetchers.get();
            int count = 0;
            while (pendingHosts.peek() != null) {
              InputHost inputHost = null;
              try {
                inputHost = pendingHosts.take();
              } catch (InterruptedException e) {
                if (isShutdown.get()) {
                  LOG.info("Interrupted and hasBeenShutdown, Breaking out of BroadcastScheduler Loop");
                  break;
                } else {
                  throw e;
                }
              }
              if (LOG.isDebugEnabled()) {
                LOG.debug("Processing pending host: " + inputHost.toDetailedString());
              }
              if (inputHost.getNumPendingInputs() > 0) {
                LOG.info("Scheduling fetch for inputHost: " + inputHost.getHost());
                Fetcher fetcher = constructFetcherForHost(inputHost);
                numRunningFetchers.incrementAndGet();
                if (isShutdown.get()) {
                  LOG.info("hasBeenShutdown, Breaking out of BroadcastScheduler Loop");
                }
                ListenableFuture<FetchResult> future = fetcherExecutor
                    .submit(fetcher);
                Futures.addCallback(future, fetchFutureCallback);
                if (++count >= maxFetchersToRun) {
                  break;
                }
              } else {
                if (LOG.isDebugEnabled()) {
                  LOG.debug("Skipping host: " + inputHost.getHost()
                      + " since it has no inputs to process");
                }
              }
            }
          } finally {

View Full Code Here

Examples of org.apache.tez.runtime.library.shuffle.common.InputHost

  
  /////////////////// Methods for InputEventHandler
  
  public void addKnownInput(String hostName, int port,
      InputAttemptIdentifier srcAttemptIdentifier, int partition) {
    InputHost host = knownSrcHosts.get(hostName);
    if (host == null) {
      host = new InputHost(hostName, port, inputContext.getApplicationId());
      InputHost old = knownSrcHosts.putIfAbsent(hostName, host);
      if (old != null) {
        host = old;
      }
    }
    if (LOG.isDebugEnabled()) {

View Full Code Here

Examples of org.apache.tez.runtime.library.shuffle.common.InputHost

          lock.lock();
          try {
            int maxFetchersToRun = numFetchers - numRunningFetchers.get();
            int count = 0;
            while (pendingHosts.peek() != null) {
              InputHost inputHost = pendingHosts.take();
              if (LOG.isDebugEnabled()) {
                LOG.debug("Processing pending host: " + inputHost.toDetailedString());
              }
              if (inputHost.getNumPendingInputs() > 0) {
                LOG.info("Scheduling fetch for inputHost: " + inputHost.getHost());
                Fetcher fetcher = constructFetcherForHost(inputHost);
                numRunningFetchers.incrementAndGet();
                ListenableFuture<FetchResult> future = fetcherExecutor
                    .submit(fetcher);
                Futures.addCallback(future, fetchFutureCallback);
                if (++count >= maxFetchersToRun) {
                  break;
                }
              } else {
                if (LOG.isDebugEnabled()) {
                  LOG.debug("Skipping host: " + inputHost.getHost()
                      + " since it has no inputs to process");
                }
              }
            }
          } finally {

View Full Code Here

Examples of org.apache.tez.runtime.library.shuffle.common.InputHost

    
    @Override
    public void onSuccess(FetchResult result) {
      Iterable<InputAttemptIdentifier> pendingInputs = result.getPendingInputs();
      if (pendingInputs != null && pendingInputs.iterator().hasNext()) {
        InputHost inputHost = knownSrcHosts.get(result.getHost());
        assert inputHost != null;
        for (InputAttemptIdentifier input : pendingInputs) {
          inputHost.addKnownInput(input);
        }
        pendingHosts.add(inputHost);
      }
      doBookKeepingForFetcherComplete();
    }

View Full Code Here

Examples of org.apache.tez.runtime.library.shuffle.common.InputHost

  /////////////////// Methods for InputEventHandler
  
  public void addKnownInput(String hostName, int port,
      InputAttemptIdentifier srcAttemptIdentifier, int srcPhysicalIndex) {
    String identifier = InputHost.createIdentifier(hostName, port);
    InputHost host = knownSrcHosts.get(identifier);
    if (host == null) {
      host = new InputHost(hostName, port, inputContext.getApplicationId(), srcPhysicalIndex);
      assert identifier.equals(host.getIdentifier());
      InputHost old = knownSrcHosts.putIfAbsent(identifier, host);
      if (old != null) {
        host = old;
      }
    }
    if (LOG.isDebugEnabled()) {

View Full Code Here

0 1

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.