Package eu.stratosphere.nephele.executiongraph

Examples of eu.stratosphere.nephele.executiongraph.ExecutionGraph


        jobRunsWithProfiling = true;
      }
 
      // Try to create initial execution graph from job graph
      LOG.info("Creating initial execution graph from job graph " + job.getName());
      ExecutionGraph eg;
 
      try {
        eg = new ExecutionGraph(job, this.instanceManager);
      } catch (GraphConversionException e) {
        if (e.getCause() == null) {
          return new JobSubmissionResult(AbstractJobResult.ReturnCode.ERROR, StringUtils.stringifyException(e));
        } else {
          Throwable t = e.getCause();
          if (t instanceof FileNotFoundException) {
            return new JobSubmissionResult(AbstractJobResult.ReturnCode.ERROR, t.getMessage());
          } else {
            return new JobSubmissionResult(AbstractJobResult.ReturnCode.ERROR, StringUtils.stringifyException(t));
          }
        }
      }
 
      // Register job with the progress collector
      if (this.eventCollector != null) {
        this.eventCollector.registerJob(eg, jobRunsWithProfiling, System.currentTimeMillis());
      }
 
      // Check if profiling should be enabled for this job
      if (jobRunsWithProfiling) {
        this.profiler.registerProfilingJob(eg);
 
        if (this.eventCollector != null) {
          this.profiler.registerForProfilingData(eg.getJobID(), this.eventCollector);
        }
 
      }
 
      // Register job with the dynamic input split assigner
      this.inputSplitManager.registerJob(eg);
 
      // Register for updates on the job status
      eg.registerJobStatusListener(this);
 
      // Schedule job
      if (LOG.isInfoEnabled()) {
        LOG.info("Scheduling job " + job.getName());
      }
View Full Code Here


    if (executionState.getExecutionState() == ExecutionState.FAILED) {
      LOG.error(executionState.getDescription());
    }

    final ExecutionGraph eg = this.scheduler.getExecutionGraphByID(executionState.getJobID());
    if (eg == null) {
      LOG.error("Cannot find execution graph for ID " + executionState.getJobID() + " to change state to "
        + executionState.getExecutionState());
      return;
    }

    final ExecutionVertex vertex = eg.getVertexByID(executionState.getID());
    if (vertex == null) {
      LOG.error("Cannot find vertex with ID " + executionState.getID() + " of job " + eg.getJobID()
        + " to change state to " + executionState.getExecutionState());
      return;
    }

    // Asynchronously update execute state of vertex
View Full Code Here

  @Override
  public JobCancelResult cancelJob(final JobID jobID) throws IOException {

    LOG.info("Trying to cancel job with ID " + jobID);

    final ExecutionGraph eg = this.scheduler.getExecutionGraphByID(jobID);
    if (eg == null) {
      return new JobCancelResult(ReturnCode.ERROR, "Cannot find job with ID " + jobID);
    }

    final Runnable cancelJobRunnable = new Runnable() {

      @Override
      public void run() {
        eg.updateJobStatus(InternalJobStatus.CANCELING, "Job canceled by user");
        final TaskCancelResult cancelResult = cancelJob(eg);
        if (cancelResult != null) {
          LOG.error(cancelResult.getDescription());
        }
      }
    };

    eg.executeCommand(cancelJobRunnable);

    LOG.info("Cancel of job " + jobID + " successfully triggered");

    return new JobCancelResult(AbstractJobResult.ReturnCode.SUCCESS, null);
  }
View Full Code Here


  @Override
  public ConnectionInfoLookupResponse lookupConnectionInfo(InstanceConnectionInfo caller, JobID jobID, ChannelID sourceChannelID) {

    final ExecutionGraph eg = this.scheduler.getExecutionGraphByID(jobID);
    if (eg == null) {
      LOG.error("Cannot find execution graph to job ID " + jobID);
      return ConnectionInfoLookupResponse.createReceiverNotFound();
    }

    final InternalJobStatus jobStatus = eg.getJobStatus();
    if (jobStatus == InternalJobStatus.FAILING || jobStatus == InternalJobStatus.CANCELING) {
      return ConnectionInfoLookupResponse.createJobIsAborting();
    }

    final ExecutionEdge edge = eg.getEdgeByID(sourceChannelID);
    if (edge == null) {
      LOG.error("Cannot find execution edge associated with ID " + sourceChannelID);
      return ConnectionInfoLookupResponse.createReceiverNotFound();
    }

    if (sourceChannelID.equals(edge.getInputChannelID())) {
      // Request was sent from an input channel
      final ExecutionVertex connectedVertex = edge.getOutputGate().getVertex();

      final AbstractInstance assignedInstance = connectedVertex.getAllocatedResource().getInstance();
      if (assignedInstance == null) {
        LOG.error("Cannot resolve lookup: vertex found for channel ID " + edge.getOutputGateIndex()
          + " but no instance assigned");
        // LOG.info("Created receiverNotReady for " + connectedVertex + " 1");
        return ConnectionInfoLookupResponse.createReceiverNotReady();
      }

      // Check execution state
      final ExecutionState executionState = connectedVertex.getExecutionState();
      if (executionState == ExecutionState.FINISHED) {
        // that should not happen. if there is data pending, the receiver cannot be ready
        return ConnectionInfoLookupResponse.createReceiverNotFound();
      }

      // running is common, finishing is happens when the lookup is for the close event
      if (executionState != ExecutionState.RUNNING && executionState != ExecutionState.FINISHING) {
        // LOG.info("Created receiverNotReady for " + connectedVertex + " in state " + executionState + " 2");
        return ConnectionInfoLookupResponse.createReceiverNotReady();
      }

      if (assignedInstance.getInstanceConnectionInfo().equals(caller)) {
        // Receiver runs on the same task manager
        return ConnectionInfoLookupResponse.createReceiverFoundAndReady(edge.getOutputChannelID());
      } else {
        // Receiver runs on a different task manager
        final InstanceConnectionInfo ici = assignedInstance.getInstanceConnectionInfo();
        final InetSocketAddress isa = new InetSocketAddress(ici.address(), ici.dataPort());

        return ConnectionInfoLookupResponse.createReceiverFoundAndReady(new RemoteReceiver(isa, edge.getConnectionID()));
      }
    }
    // else, the request is for an output channel
    // Find vertex of connected input channel
    final ExecutionVertex targetVertex = edge.getInputGate().getVertex();

    // Check execution state
    final ExecutionState executionState = targetVertex.getExecutionState();

    // check whether the task needs to be deployed
    if (executionState != ExecutionState.RUNNING && executionState != ExecutionState.FINISHING && executionState != ExecutionState.FINISHED) {

      if (executionState == ExecutionState.ASSIGNED) {
        final Runnable command = new Runnable() {
          @Override
          public void run() {
            scheduler.deployAssignedVertices(targetVertex);
          }
        };
        eg.executeCommand(command);
      }

      // LOG.info("Created receiverNotReady for " + targetVertex + " in state " + executionState + " 3");
      return ConnectionInfoLookupResponse.createReceiverNotReady();
    }
View Full Code Here

  }

  @Override
  public void killTask(final JobID jobID, final ManagementVertexID id) throws IOException {

    final ExecutionGraph eg = this.scheduler.getExecutionGraphByID(jobID);
    if (eg == null) {
      LOG.error("Cannot find execution graph for job " + jobID);
      return;
    }

    final ExecutionVertex vertex = eg.getVertexByID(ExecutionVertexID.fromManagementVertexID(id));
    if (vertex == null) {
      LOG.error("Cannot find execution vertex with ID " + id);
      return;
    }

    LOG.info("Killing task " + vertex + " of job " + jobID);

    final Runnable runnable = new Runnable() {

      @Override
      public void run() {

        final TaskKillResult result = vertex.killTask();
        if (result.getReturnCode() != AbstractTaskResult.ReturnCode.SUCCESS) {
          LOG.error(result.getDescription());
        }
      }
    };

    eg.executeCommand(runnable);
  }
View Full Code Here


  @Override
  public void logBufferUtilization(final JobID jobID) throws IOException {

    final ExecutionGraph eg = this.scheduler.getExecutionGraphByID(jobID);
    if (eg == null) {
      return;
    }

    final Set<AbstractInstance> allocatedInstance = new HashSet<AbstractInstance>();
View Full Code Here

  @Override
  public InputSplitWrapper requestNextInputSplit(final JobID jobID, final ExecutionVertexID vertexID,
      final IntegerRecord sequenceNumber) throws IOException {

    final ExecutionGraph graph = this.scheduler.getExecutionGraphByID(jobID);
    if (graph == null) {
      LOG.error("Cannot find execution graph to job ID " + jobID);
      return null;
    }

    final ExecutionVertex vertex = graph.getVertexByID(vertexID);
    if (vertex == null) {
      LOG.error("Cannot find execution vertex for vertex ID " + vertexID);
      return null;
    }
View Full Code Here

      fail(StringUtils.stringifyException(e));
    }

    try {
      LibraryCacheManager.register(jobGraph.getJobID(), new String[0]);
      return new ExecutionGraph(jobGraph, instanceManager);

    } catch (GraphConversionException e) {
      fail(StringUtils.stringifyException(e));
    } catch (IOException e) {
      fail(StringUtils.stringifyException(e));
View Full Code Here

    final TestInstanceManager tim = new TestInstanceManager();
    final TestDeploymentManager tdm = new TestDeploymentManager();
    final QueueScheduler scheduler = new QueueScheduler(tdm, tim);

    final ExecutionGraph executionGraph = createExecutionGraph(ChannelType.IN_MEMORY, tim);

    try {
      try {
        scheduler.schedulJob(executionGraph);
      } catch (SchedulingException e) {
        fail(StringUtils.stringifyException(e));
      }

      // Wait for the deployment to complete
      tdm.waitForDeployment();

      assertEquals(executionGraph.getJobID(), tdm.getIDOfLastDeployedJob());
      final List<ExecutionVertex> listOfDeployedVertices = tdm.getListOfLastDeployedVertices();
      assertNotNull(listOfDeployedVertices);
      // Vertices connected via in-memory channels must be deployed in a single cycle.
      assertEquals(2, listOfDeployedVertices.size());

      // Check if the release of the allocated resources works properly by simulating the vertices' life cycle
      assertEquals(0, tim.getNumberOfReleaseMethodCalls());

      // Simulate vertex life cycle
      for (final ExecutionVertex vertex : listOfDeployedVertices) {
        vertex.updateExecutionState(ExecutionState.STARTING);
        vertex.updateExecutionState(ExecutionState.RUNNING);
        vertex.updateExecutionState(ExecutionState.FINISHING);
        vertex.updateExecutionState(ExecutionState.FINISHED);
      }

      assertEquals(1, tim.getNumberOfReleaseMethodCalls());
    } finally {
      try {
        LibraryCacheManager.unregister(executionGraph.getJobID());
      } catch (IOException ioe) {
        // Ignore exception here
      }
    }
  }
View Full Code Here

    synchronized (this.jobQueue) {

      final Iterator<ExecutionGraph> it = this.jobQueue.iterator();
      while (it.hasNext()) {

        final ExecutionGraph executionGraph = it.next();
        // Field jobID of executionGraph is immutable, so no synchronization needed
        if (executionGraph.getJobID().equals(executionGraphToRemove.getJobID())) {
          removedFromQueue = true;
          it.remove();
          break;
        }
View Full Code Here

TOP

Related Classes of eu.stratosphere.nephele.executiongraph.ExecutionGraph

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.