Examples of org.apache.tez.dag.api.client.DAGClient

org.apache.tez.dag.api.client.DAGClient
Class for monitoring the DAG running in a Tez DAG Application Master.

      tezSession.start();
      LOG.info("Created Tez Session");
    }


    DAGStatus dagStatus = null;
    DAGClient dagClient = null;
    String[] vNames = { "initialmap", "intermediate_reducer",
      "finalreduce" };


    Set<StatusGetOpts> statusGetOpts = EnumSet.of(StatusGetOpts.GET_COUNTERS);
    try {
      for (int dagIndex = 1; dagIndex <= inputPaths.size(); ++dagIndex) {
        if (dagIndex != 1
            && interJobSleepTimeout > 0) {
          try {
            LOG.info("Sleeping between jobs, sleepInterval="
                + (interJobSleepTimeout/1000));
            Thread.sleep(interJobSleepTimeout);
          } catch (InterruptedException e) {
            LOG.info("Main thread interrupted. Breaking out of job loop");
            break;
          }
        }


        String inputPath = inputPaths.get(dagIndex-1);
        String outputPath = outputPaths.get(dagIndex-1);


        if (fs.exists(new Path(outputPath))) {
          throw new FileAlreadyExistsException("Output directory "
              + outputPath + " already exists");
        }
        LOG.info("Running OrderedWordCount DAG"
            + ", dagIndex=" + dagIndex
            + ", inputPath=" + inputPath
            + ", outputPath=" + outputPath);


        Map<String, LocalResource> localResources =
          new TreeMap<String, LocalResource>();
        
        DAG dag = instance.createDAG(fs, conf, localResources,
            stagingDir, dagIndex, inputPath, outputPath,
            generateSplitsInClient);


        boolean doPreWarm = dagIndex == 1 && useTezSession
            && conf.getBoolean("PRE_WARM_SESSION", true);
        int preWarmNumContainers = 0;
        if (doPreWarm) {
          preWarmNumContainers = conf.getInt("PRE_WARM_NUM_CONTAINERS", 0);
          if (preWarmNumContainers <= 0) {
            doPreWarm = false;
          }
        }
        if (doPreWarm) {
          LOG.info("Pre-warming Session");
          VertexLocationHint vertexLocationHint =
              new VertexLocationHint(null);
          ProcessorDescriptor sleepProcDescriptor =
            new ProcessorDescriptor(SleepProcessor.class.getName());
          SleepProcessor.SleepProcessorConfig sleepProcessorConfig =
            new SleepProcessor.SleepProcessorConfig(4000);
          sleepProcDescriptor.setUserPayload(
            sleepProcessorConfig.toUserPayload());
          PreWarmContext context = new PreWarmContext(sleepProcDescriptor,
            dag.getVertex("initialmap").getTaskResource(), preWarmNumContainers,
              vertexLocationHint);


          Map<String, LocalResource> contextLocalRsrcs =
            new TreeMap<String, LocalResource>();
          contextLocalRsrcs.putAll(
            dag.getVertex("initialmap").getTaskLocalResources());
          Map<String, String> contextEnv = new TreeMap<String, String>();
          contextEnv.putAll(dag.getVertex("initialmap").getTaskEnvironment());
          String contextJavaOpts =
            dag.getVertex("initialmap").getJavaOpts();
          context
            .setLocalResources(contextLocalRsrcs)
            .setJavaOpts(contextJavaOpts)
            .setEnvironment(contextEnv);


          tezSession.preWarm(context);
        }


        if (useTezSession) {
          LOG.info("Waiting for TezSession to get into ready state");
          waitForTezSessionReady(tezSession);
          LOG.info("Submitting DAG to Tez Session, dagIndex=" + dagIndex);
          dagClient = tezSession.submitDAG(dag);
          LOG.info("Submitted DAG to Tez Session, dagIndex=" + dagIndex);
        } else {
          LOG.info("Submitting DAG as a new Tez Application");
          dagClient = tezClient.submitDAGApplication(dag, amConfig);
        }


        while (true) {
          dagStatus = dagClient.getDAGStatus(statusGetOpts);
          if (dagStatus.getState() == DAGStatus.State.RUNNING ||
              dagStatus.getState() == DAGStatus.State.SUCCEEDED ||
              dagStatus.getState() == DAGStatus.State.FAILED ||
              dagStatus.getState() == DAGStatus.State.KILLED ||
              dagStatus.getState() == DAGStatus.State.ERROR) {
            break;
          }
          try {
            Thread.sleep(500);
          } catch (InterruptedException e) {
            // continue;
          }
        }




        while (dagStatus.getState() != DAGStatus.State.SUCCEEDED &&
            dagStatus.getState() != DAGStatus.State.FAILED &&
            dagStatus.getState() != DAGStatus.State.KILLED &&
            dagStatus.getState() != DAGStatus.State.ERROR) {
          if (dagStatus.getState() == DAGStatus.State.RUNNING) {
            ExampleDriver.printDAGStatus(dagClient, vNames);
          }
          try {
            try {
              Thread.sleep(1000);
            } catch (InterruptedException e) {
              // continue;
            }
            dagStatus = dagClient.getDAGStatus(statusGetOpts);
          } catch (TezException e) {
            LOG.fatal("Failed to get application progress. Exiting");
            System.exit(-1);
          }
        }

View Full Code Here

        MRHelpers.getMRAMJavaOpts(conf));


    AMConfiguration amConfig = new AMConfiguration(null,
        null, conf, this.credentials);


    DAGClient dagClient =
        tezClient.submitDAGApplication(appId, dag, amConfig);


    while (true) {
      DAGStatus status = dagClient.getDAGStatus(null);
      LOG.info("DAG Status: " + status);
      if (status.isCompleted()) {
        break;
      }
      try {
        Thread.sleep(1000);
      } catch (InterruptedException e) {
        // do nothing
      }
    }


    return dagClient.getDAGStatus(null).getState().equals(DAGStatus.State.SUCCEEDED) ? 0 : 1;
  }

View Full Code Here

      status = tezSession.getSessionStatus();
    }
    if (status == TezSessionStatus.SHUTDOWN) {
      throw new TezUncheckedException("Unexpected Session shutdown");
    }
    DAGClient dagClient = tezSession.submitDAG(dag);
    DAGStatus dagStatus = dagClient.getDAGStatus(null);
    while (!dagStatus.isCompleted()) {
      LOG.info("Waiting for dag to complete. Sleeping for 500ms."
          + " DAG name: " + dag.getName()
          + " DAG appId: " + dagClient.getApplicationId()
          + " Current state: " + dagStatus.getState());
      Thread.sleep(100);
      dagStatus = dagClient.getDAGStatus(null);
    }
    
    Assert.assertEquals(finalState, dagStatus.getState());
  }

View Full Code Here

      status = session.getSessionStatus();
    }
    if (status == TezSessionStatus.SHUTDOWN) {
      throw new TezUncheckedException("Unexpected Session shutdown");
    }
    DAGClient dagClient = session.submitDAG(dag);
    DAGStatus dagStatus = dagClient.getDAGStatus(null);
    while (!dagStatus.isCompleted()) {
      LOG.info("Waiting for dag to complete. Sleeping for 500ms."
          + " DAG name: " + dag.getName()
          + " DAG appId: " + dagClient.getApplicationId()
          + " Current state: " + dagStatus.getState());
      Thread.sleep(100);
      dagStatus = dagClient.getDAGStatus(null);
    }


    Assert.assertEquals(finalState, dagStatus.getState());
  }

View Full Code Here


    TezClient tezClient = new TezClient(tezConf);
    AMConfiguration amConf = new AMConfiguration(new HashMap<String, String>(),
        new HashMap<String, LocalResource>(), tezConf, null);


    DAGClient dagClient = tezClient.submitDAGApplication(dag, amConf);


    DAGStatus dagStatus = dagClient.getDAGStatus(null);
    while (!dagStatus.isCompleted()) {
      LOG.info("Waiting for job to complete. Sleeping for 500ms." + " Current state: "
          + dagStatus.getState());
      Thread.sleep(500l);
      dagStatus = dagClient.getDAGStatus(null);
    }
    dagStatus = dagClient.getDAGStatus(Sets.newHashSet(StatusGetOpts.GET_COUNTERS));


    assertEquals(DAGStatus.State.SUCCEEDED, dagStatus.getState());
    assertNotNull(dagStatus.getDAGCounters());
    assertNotNull(dagStatus.getDAGCounters().getGroup(FileSystemCounter.class.getName()));
    assertNotNull(dagStatus.getDAGCounters().findCounter(TaskCounter.GC_TIME_MILLIS));

View Full Code Here


    TezClient tezClient = new TezClient(tezConf);
    AMConfiguration amConf = new AMConfiguration(new HashMap<String, String>(),
        new HashMap<String, LocalResource>(), tezConf, null);


    DAGClient dagClient = tezClient.submitDAGApplication(dag, amConf);


    DAGStatus dagStatus = dagClient.getDAGStatus(null);
    while (!dagStatus.isCompleted()) {
      LOG.info("Waiting for job to complete. Sleeping for 500ms." + " Current state: "
          + dagStatus.getState());
      Thread.sleep(500l);
      dagStatus = dagClient.getDAGStatus(null);
    }
    dagStatus = dagClient.getDAGStatus(Sets.newHashSet(StatusGetOpts.GET_COUNTERS));


    assertEquals(DAGStatus.State.SUCCEEDED, dagStatus.getState());
    assertNotNull(dagStatus.getDAGCounters());
    assertNotNull(dagStatus.getDAGCounters().getGroup(FileSystemCounter.class.getName()));
    assertNotNull(dagStatus.getDAGCounters().findCounter(TaskCounter.GC_TIME_MILLIS));

View Full Code Here

            OnFileUnorderedKVOutput.class.getName()), new InputDescriptor(
            ShuffledUnorderedKVInput.class.getName())));
    dag.addVertex(stage1Vertex).addVertex(stage2Vertex).addEdge(edge);


    LOG.info("Submitting DAG to Tez Session");
    DAGClient dagClient = tezSession.submitDAG(dag);
    LOG.info("Submitted DAG to Tez Session");


    DAGStatus dagStatus = null;
    String[] vNames = { "stage1", "stage2" };
    try {
      while (true) {
        dagStatus = dagClient.getDAGStatus(null);
        if(dagStatus.getState() == DAGStatus.State.RUNNING ||
            dagStatus.getState() == DAGStatus.State.SUCCEEDED ||
            dagStatus.getState() == DAGStatus.State.FAILED ||
            dagStatus.getState() == DAGStatus.State.KILLED ||
            dagStatus.getState() == DAGStatus.State.ERROR) {
          break;
        }
        try {
          Thread.sleep(500);
        } catch (InterruptedException e) {
          // continue;
        }
      }


      while (dagStatus.getState() == DAGStatus.State.RUNNING) {
        try {
          ExampleDriver.printDAGStatus(dagClient, vNames);
          try {
            Thread.sleep(1000);
          } catch (InterruptedException e) {
            // continue;
          }
          dagStatus = dagClient.getDAGStatus(null);
        } catch (TezException e) {
          LOG.fatal("Failed to get application progress. Exiting");
          System.exit(-1);
        }
      }

View Full Code Here

  @Override
  public int execute(DriverContext driverContext) {
    int rc = 1;
    boolean cleanContext = false;
    Context ctx = null;
    DAGClient client = null;
    TezSessionState session = null;


    try {
      // Get or create Context object. If we create it we have to clean it later as well.
      ctx = driverContext.getCtx();
      if (ctx == null) {
        ctx = new Context(conf);
        cleanContext = true;
      }


      // Need to remove this static hack. But this is the way currently to get a session.
      SessionState ss = SessionState.get();
      session = ss.getTezSession();
      session = TezSessionPoolManager.getInstance().getSession(session, conf, false);
      ss.setTezSession(session);


      // jobConf will hold all the configuration for hadoop, tez, and hive
      JobConf jobConf = utils.createConfiguration(conf);


      // Get all user jars from work (e.g. input format stuff).
      String[] inputOutputJars = work.configureJobConfAndExtractJars(jobConf);


      // we will localize all the files (jars, plans, hashtables) to the
      // scratch dir. let's create this and tmp first.
      Path scratchDir = ctx.getMRScratchDir();


      // create the tez tmp dir
      scratchDir = utils.createTezDir(scratchDir, conf);
      boolean hasResources = session.hasResources(inputOutputJars);


      if (ss.hasAddedResource()) {
        // need to re-launch session because of new added jars.
        hasResources = false;
        // reset the added resource flag for this session since we would
        // relocalize (either by restarting or relocalizing) due to the above
        // hasResources flag.
        ss.setAddedResource(false);
      }


      // If we have any jars from input format, we need to restart the session because
      // AM will need them; so, AM has to be restarted. What a mess...
      if (!hasResources && session.isOpen()) {
        LOG.info("Tez session being reopened to pass custom jars to AM");
        TezSessionPoolManager.getInstance().close(session);
        session = TezSessionPoolManager.getInstance().getSession(null, conf, false);
        ss.setTezSession(session);
      }


      if (!session.isOpen()) {
        // can happen if the user sets the tez flag after the session was
        // established
        LOG.info("Tez session hasn't been created yet. Opening session");
        session.open(conf, inputOutputJars);
      }
      List<LocalResource> additionalLr = session.getLocalizedResources();


      // unless already installed on all the cluster nodes, we'll have to
      // localize hive-exec.jar as well.
      LocalResource appJarLr = session.getAppJarLr();


      // next we translate the TezWork to a Tez DAG
      DAG dag = build(jobConf, work, scratchDir, appJarLr, additionalLr, ctx);


      // submit will send the job to the cluster and start executing
      client = submit(jobConf, dag, scratchDir, appJarLr, session);


      // finally monitor will print progress until the job is done
      TezJobMonitor monitor = new TezJobMonitor();
      rc = monitor.monitorExecution(client, ctx.getHiveTxnManager(), conf);


      // fetch the counters
      Set<StatusGetOpts> statusGetOpts = EnumSet.of(StatusGetOpts.GET_COUNTERS);
      counters = client.getDAGStatus(statusGetOpts).getDAGCounters();
      TezSessionPoolManager.getInstance().returnSession(session);


      if (LOG.isInfoEnabled()) {
        for (CounterGroup group: counters) {
          LOG.info(group.getDisplayName() +":");

View Full Code Here

      LocalResource appJarLr, TezSessionState sessionState)
      throws IOException, TezException, InterruptedException,
      LoginException, URISyntaxException, HiveException {


    perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.TEZ_SUBMIT_DAG);
    DAGClient dagClient = null;


    try {
      // ready to start execution on the cluster
      dagClient = sessionState.getSession().submitDAG(dag);
    } catch (SessionNotRunning nr) {

View Full Code Here

  @Override
  public int execute(DriverContext driverContext) {
    int rc = 1;
    boolean cleanContext = false;
    Context ctx = null;
    DAGClient client = null;
    TezSessionState session = null;


    try {
      // Get or create Context object. If we create it we have to clean it later as well.
      ctx = driverContext.getCtx();
      if (ctx == null) {
        ctx = new Context(conf);
        cleanContext = true;
      }


      // Need to remove this static hack. But this is the way currently to get a session.
      SessionState ss = SessionState.get();
      session = ss.getTezSession();
      session = TezSessionPoolManager.getInstance().getSession(session, conf, false);
      ss.setTezSession(session);


      // jobConf will hold all the configuration for hadoop, tez, and hive
      JobConf jobConf = utils.createConfiguration(conf);


      // Get all user jars from work (e.g. input format stuff).
      String[] inputOutputJars = work.configureJobConfAndExtractJars(jobConf);


      // we will localize all the files (jars, plans, hashtables) to the
      // scratch dir. let's create this and tmp first.
      Path scratchDir = ctx.getMRScratchDir();


      // create the tez tmp dir
      scratchDir = utils.createTezDir(scratchDir, conf);


      Map<String,LocalResource> inputOutputLocalResources =
          getExtraLocalResources(jobConf, scratchDir, inputOutputJars);


      // Ensure the session is open and has the necessary local resources
      updateSession(session, jobConf, scratchDir, inputOutputJars, inputOutputLocalResources);


      List<LocalResource> additionalLr = session.getLocalizedResources();


      // log which resources we're adding (apart from the hive exec)
      if (LOG.isDebugEnabled()) {
        if (additionalLr == null || additionalLr.size() == 0) {
          LOG.debug("No local resources to process (other than hive-exec)");
        } else {
          for (LocalResource lr: additionalLr) {
            LOG.debug("Adding local resource: " + lr.getResource());
          }
        }
      }


      // unless already installed on all the cluster nodes, we'll have to
      // localize hive-exec.jar as well.
      LocalResource appJarLr = session.getAppJarLr();


      // next we translate the TezWork to a Tez DAG
      DAG dag = build(jobConf, work, scratchDir, appJarLr, additionalLr, ctx);


      // Add the extra resources to the dag
      addExtraResourcesToDag(session, dag, inputOutputJars, inputOutputLocalResources);


      // submit will send the job to the cluster and start executing
      client = submit(jobConf, dag, scratchDir, appJarLr, session,
          additionalLr, inputOutputJars, inputOutputLocalResources);


      // finally monitor will print progress until the job is done
      TezJobMonitor monitor = new TezJobMonitor();
      rc = monitor.monitorExecution(client, ctx.getHiveTxnManager(), conf, dag);


      // fetch the counters
      Set<StatusGetOpts> statusGetOpts = EnumSet.of(StatusGetOpts.GET_COUNTERS);
      counters = client.getDAGStatus(statusGetOpts).getDAGCounters();
      TezSessionPoolManager.getInstance().returnSession(session);


      if (LOG.isInfoEnabled() && counters != null
          && conf.getBoolVar(conf, HiveConf.ConfVars.TEZ_EXEC_SUMMARY)) {
        for (CounterGroup group: counters) {

View Full Code Here

0 1 2 3 4 5 6

TOP

Related Classes of org.apache.tez.dag.api.client.DAGClient

cascading.stats.tez.TezNodeStats

cascading.stats.tez.TezStepStats

org.apache.hadoop.hive.ql.exec.tez.TezTask

org.apache.tez.client.TestLocalMode

org.apache.tez.client.TestTezClient

org.apache.tez.examples.JoinDataGen

org.apache.tez.examples.JoinExample

org.apache.tez.examples.JoinValidate

org.apache.tez.examples.OrderedWordCount

org.apache.tez.examples.SimpleSessionExample

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.