Examples of TezConfiguration

org.apache.tez.dag.api.TezConfiguration
Defines the configurations for Tez. These configurations are typically specified in tez-site.xml on the client machine where TezClient is used to launch the Tez application. tez-site.xml is expected to be picked up from the classpath of the client process.

Examples of org.apache.tez.dag.api.TezConfiguration

  private static String HOST = "localhost";
  private static int PORT = 0;


  @Test(timeout = 3000)
  public void testLocalFetchModeSetting() throws Exception {
    TezConfiguration conf = new TezConfiguration();
    conf.set(TezRuntimeConfiguration.TEZ_RUNTIME_OPTIMIZE_LOCAL_FETCH, "true");
    EnvironmentUpdateUtils.put(ApplicationConstants.Environment.NM_HOST.toString(), HOST);
    InputAttemptIdentifier[] srcAttempts = {
        new InputAttemptIdentifier(0, 1, InputAttemptIdentifier.PATH_PREFIX + "pathComponent_1")
    };
    FetcherCallback fetcherCallback = mock(FetcherCallback.class);


    Fetcher.FetcherBuilder builder = new Fetcher.FetcherBuilder(fetcherCallback, null, null,
        ApplicationId.newInstance(0, 1), null, "fetcherTest", conf, true);
    builder.assignWork(HOST, PORT, 0, Arrays.asList(srcAttempts));
    Fetcher fetcher = spy(builder.build());


    FetchResult fr = new FetchResult(HOST, PORT, 0, Arrays.asList(srcAttempts));
    Fetcher.HostFetchResult hfr = new Fetcher.HostFetchResult(fr, srcAttempts, false);
    doReturn(hfr).when(fetcher).setupLocalDiskFetch();
    doReturn(null).when(fetcher).doHttpFetch();
    doNothing().when(fetcher).shutdown();


    fetcher.call();


    verify(fetcher).setupLocalDiskFetch();
    verify(fetcher, never()).doHttpFetch();


    // When disabled use http fetch
    conf.set(TezRuntimeConfiguration.TEZ_RUNTIME_OPTIMIZE_LOCAL_FETCH, "false");
    builder = new Fetcher.FetcherBuilder(fetcherCallback, null, null,
        ApplicationId.newInstance(0, 1), null, "fetcherTest", conf, false);
    builder.assignWork(HOST, PORT, 0, Arrays.asList(srcAttempts));
    fetcher = spy(builder.build());

View Full Code Here

Examples of org.apache.tez.dag.api.TezConfiguration

    };
    final int FIRST_FAILED_ATTEMPT_IDX = 2;
    final int SECOND_FAILED_ATTEMPT_IDX = 4;
    final int[] sucessfulAttempts = {0, 1, 3};


    TezConfiguration conf = new TezConfiguration();
    conf.set(TezRuntimeConfiguration.TEZ_RUNTIME_OPTIMIZE_LOCAL_FETCH, "true");
    EnvironmentUpdateUtils.put(ApplicationConstants.Environment.NM_HOST.toString(), HOST);
    int partition = 42;
    FetcherCallback callback = mock(FetcherCallback.class);
    Fetcher.FetcherBuilder builder = new Fetcher.FetcherBuilder(callback, null, null,
        ApplicationId.newInstance(0, 1), null, "fetcherTest", conf, true);

View Full Code Here

Examples of org.apache.tez.dag.api.TezConfiguration

    Vertex vertex = Vertex.create("SleepVertex", ProcessorDescriptor.create(
            SleepProcessor.class.getName()).setUserPayload(spConf.toUserPayload()), 1,
        Resource.newInstance(1024, 1));
    dag.addVertex(vertex);


    TezConfiguration tezConf = new TezConfiguration(mrrTezCluster.getConfig());
    Path remoteStagingDir = remoteFs.makeQualified(new Path("/tmp", String.valueOf(random
        .nextInt(100000))));
    remoteFs.mkdirs(remoteStagingDir);
    tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, remoteStagingDir.toString());


    TezClient tezSession = TezClient.create("TezSleepProcessor", tezConf, false);
    tezSession.start();


    DAGClient dagClient = tezSession.submitDAG(dag);

View Full Code Here

Examples of org.apache.tez.dag.api.TezConfiguration

  @Test(timeout = 100000)
  public void testMultipleDAGsWithDuplicateName() throws TezException, IOException,
      InterruptedException {
    TezClient tezSession = null;
    try {
      TezConfiguration tezConf = new TezConfiguration(mrrTezCluster.getConfig());
      Path remoteStagingDir = remoteFs.makeQualified(new Path("/tmp", String.valueOf(random
          .nextInt(100000))));
      remoteFs.mkdirs(remoteStagingDir);
      tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, remoteStagingDir.toString());
      tezSession = TezClient.create("OrderedWordCountSession", tezConf, true);
      tezSession.start();


      SleepProcessorConfig spConf = new SleepProcessorConfig(1);
      for (int dagIndex = 1; dagIndex <= 2; dagIndex++) {

View Full Code Here

Examples of org.apache.tez.dag.api.TezConfiguration

    Vertex vertex = Vertex.create("SleepVertex", ProcessorDescriptor.create(
            SleepProcessor.class.getName()).setUserPayload(spConf.toUserPayload()), 1,
        Resource.newInstance(1024, 1));
    dag.addVertex(vertex);


    TezConfiguration tezConf = new TezConfiguration(mrrTezCluster.getConfig());
    Path stagingDir = new Path(TEST_ROOT_DIR, "testNonDefaultFSStagingDir"
        + String.valueOf(random.nextInt(100000)));
    FileSystem localFs = FileSystem.getLocal(tezConf);
    stagingDir = localFs.makeQualified(stagingDir);
    localFs.mkdirs(stagingDir);
    tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, stagingDir.toString());


    TezClient tezSession = TezClient.create("TezSleepProcessor", tezConf, false);
    tezSession.start();


    DAGClient dagClient = tezSession.submitDAG(dag);

View Full Code Here

Examples of org.apache.tez.dag.api.TezConfiguration

    Vertex vertex = Vertex.create("SleepVertex", ProcessorDescriptor.create(
            SleepProcessor.class.getName()).setUserPayload(spConf.toUserPayload()), 2,
        Resource.newInstance(1024, 1));
    dag.addVertex(vertex);


    TezConfiguration tezConf = new TezConfiguration(mrrTezCluster.getConfig());
    Path remoteStagingDir = remoteFs.makeQualified(new Path("/tmp", String.valueOf(random
        .nextInt(100000))));
    remoteFs.mkdirs(remoteStagingDir);
    tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, remoteStagingDir.toString());


    FileSystem localFs = FileSystem.getLocal(tezConf);
    Path historyLogDir = new Path(TEST_ROOT_DIR, "testHistoryLogging");
    localFs.mkdirs(historyLogDir);


    tezConf.set(TezConfiguration.TEZ_SIMPLE_HISTORY_LOGGING_DIR,
        localFs.makeQualified(historyLogDir).toString());


    tezConf.setBoolean(TezConfiguration.TEZ_AM_SESSION_MODE, false);
    TezClient tezSession = TezClient.create("TezSleepProcessorHistoryLogging", tezConf);
    tezSession.start();


    DAGClient dagClient = tezSession.submitDAG(dag);

View Full Code Here

Examples of org.apache.tez.dag.api.TezConfiguration


  private TezClient createTezSession() throws IOException, TezException {
    Path remoteStagingDir = remoteFs.makeQualified(new Path("/tmp", String
        .valueOf(new Random().nextInt(100000))));
    remoteFs.mkdirs(remoteStagingDir);
    TezConfiguration tezConf = new TezConfiguration(mrrTezCluster.getConfig());
    tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, remoteStagingDir.toString());


    TezClient tezSession = TezClient.create("testrelocalizationsession", tezConf, true);
    tezSession.start();
    Assert.assertEquals(TezAppMasterStatus.INITIALIZING, tezSession.getAppMasterStatus());
    return tezSession;

View Full Code Here

Examples of org.apache.tez.dag.api.TezConfiguration

  public void testMultipleMRRSleepJobViaSession() throws IOException,
  InterruptedException, TezException, ClassNotFoundException, YarnException {
    Path remoteStagingDir = remoteFs.makeQualified(new Path("/tmp", String
        .valueOf(new Random().nextInt(100000))));
    remoteFs.mkdirs(remoteStagingDir);
    TezConfiguration tezConf = new TezConfiguration(
        mrrTezCluster.getConfig());
    tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR,
        remoteStagingDir.toString());


    TezClient tezSession = TezClient.create("testsession", tezConf, true);
    tezSession.start();
    Assert.assertEquals(TezAppMasterStatus.INITIALIZING,

View Full Code Here

Examples of org.apache.tez.dag.api.TezConfiguration

            OrderedGroupedInputLegacy.class.getName()).setUserPayload(stage3Payload)));


    dag.addEdge(edge1);
    dag.addEdge(edge2);


    TezConfiguration tezConf = new TezConfiguration(
            mrrTezCluster.getConfig());
    tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR,
        remoteStagingDir.toString());


    DAGClient dagClient = null;
    boolean reuseSession = reUseTezSession != null;
    TezClient tezSession = null;
    if (!dagViaRPC) {
      Preconditions.checkArgument(reuseSession == false);
    }
    if (!reuseSession) {
      TezConfiguration tempTezconf = new TezConfiguration(tezConf);
      if (!dagViaRPC) {
        tempTezconf.setBoolean(TezConfiguration.TEZ_AM_SESSION_MODE, false);
      } else {
        tempTezconf.setBoolean(TezConfiguration.TEZ_AM_SESSION_MODE, true);
      }
      tezSession = TezClient.create("testsession", tempTezconf);
      tezSession.start();
    } else {
      tezSession = reUseTezSession;

View Full Code Here

Examples of org.apache.tez.dag.api.TezConfiguration

    }
    return 0;
  }


  private int execute(String[] args) throws TezException, IOException, InterruptedException {
    TezConfiguration tezConf = new TezConfiguration(getConf());
    TezClient tezClient = null;
    try {
      tezClient = createTezClient(tezConf);
      return execute(args, tezConf, tezClient);
    } finally {

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.