Examples of TaskSpec

Examples of org.apache.tez.runtime.api.impl.TaskSpec

168
169
170
171
172
173
174
175
176
177
178
        new InputDescriptor(LocalMergedInput.class.getName()), 1);
    OutputSpec reduceOutputSpec = new OutputSpec("NullDestinationVertex",
        new OutputDescriptor(MROutputLegacy.class.getName()), 1);

    // Now run a reduce
    TaskSpec taskSpec = new TaskSpec(
        TezTestUtils.getMockTaskAttemptId(0, 1, 0, 0),
        dagName,
        reduceVertexName,
        reduceProcessorDesc,
        Collections.singletonList(reduceInputSpec),
View Full Code Here

Examples of org.apache.tez.runtime.api.impl.TaskSpec

63
64
65
66
67
68
69
70
71
72
73
74
75
76
    Multimap<String, String> startedInputsMap = HashMultimap.create();
    TezUmbilical umbilical = mock(TezUmbilical.class);
    TezConfiguration tezConf = new TezConfiguration();

    TezTaskAttemptID taId1 = createTaskAttemptID(vertexId, 1);
    TaskSpec task1 = createTaskSpec(taId1, "dag1", "vertex1");

    TezTaskAttemptID taId2 = createTaskAttemptID(vertexId, 2);
    TaskSpec task2 = createTaskSpec(taId2, "dag2", "vertex1");

    LogicalIOProcessorRuntimeTask lio1 = new LogicalIOProcessorRuntimeTask(task1, 0, tezConf,
        umbilical, serviceConsumerMetadata, startedInputsMap);

    lio1.initialize();
View Full Code Here

Examples of org.apache.tez.runtime.api.impl.TaskSpec

97
98
99
100
101
102
103
104
105
106
  }

  private TaskSpec createTaskSpec(TezTaskAttemptID taskAttemptID,
      String dagName, String vertexName) {
    ProcessorDescriptor processorDesc = createProcessorDescriptor();
    TaskSpec taskSpec = new TaskSpec(taskAttemptID,
        dagName, vertexName, processorDesc,
        createInputSpecList(), createOutputSpecList(), null);
    return taskSpec;
  }
View Full Code Here

Examples of org.apache.tez.runtime.api.impl.TaskSpec

201
202
203
204
205
206
207
208
209
210
211
    ProcessorDescriptor mapProcessorDesc = new ProcessorDescriptor(
        MapProcessor.class.getName()).setUserPayload(TezUtils.createUserPayloadFromConf(jobConf));
   
    Token<JobTokenIdentifier> shuffleToken = new Token<JobTokenIdentifier>();

    TaskSpec taskSpec = new TaskSpec(
        TezTestUtils.getMockTaskAttemptId(0, 0, mapId, 0),
        dagName, vertexName,
        mapProcessorDesc,
        inputSpecs,
        outputSpecs, null);
View Full Code Here

Examples of org.apache.tez.runtime.api.impl.TaskSpec

503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
              }
            }), defaultConf);
        RelocalizationUtils.addUrlsToClassPath(downloadedUrls);

        LOG.info("Done localizing additional resources");
        final TaskSpec taskSpec = containerTask.getTaskSpec();
        if (LOG.isDebugEnabled()) {
          LOG.debug("New container task context:"
              + taskSpec.toString());
        }

        try {
          taskLock.writeLock().lock();
          currentTaskAttemptID = taskSpec.getTaskAttemptID();
          TezVertexID newVertexId =
              currentTaskAttemptID.getTaskID().getVertexID();
          currentTaskComplete.set(false);

          if (lastVertexId != null) {
            if (!lastVertexId.equals(newVertexId)) {
              objectRegistry.clearCache(ObjectLifeCycle.VERTEX);
            }
            if (!lastVertexId.getDAGId().equals(newVertexId.getDAGId())) {
              objectRegistry.clearCache(ObjectLifeCycle.DAG);
              startedInputsMap = HashMultimap.create();
            }
          }
          lastVertexId = newVertexId;
          TezUtils.updateLoggers(currentTaskAttemptID.toString());
          loggerAddend = currentTaskAttemptID.toString() + "_post";
         
          currentTask = createLogicalTask(attemptNumber, taskSpec,
              defaultConf, tezUmbilical, serviceConsumerMetadata);
         
          taskNonOobHeartbeatCounter = 0;
          nextHeartbeatNumToLog = (Math.max(1,
              (int) (LOG_COUNTER_START_INTERVAL / (amPollInterval == 0 ? 0.000001f
                  : (float) amPollInterval))));
        } finally {
          taskLock.writeLock().unlock();
        }

        final EventMetaData sourceInfo = new EventMetaData(
            EventProducerConsumerType.SYSTEM,
            taskSpec.getVertexName(), "", currentTaskAttemptID);
        currentSourceInfo = sourceInfo;

        // TODO Initiate Java VM metrics
        // JvmMetrics.initSingleton(containerId.toString(), job.getSessionId());
View Full Code Here

Examples of org.apache.tez.runtime.api.impl.TaskSpec

899
900
901
902
903
904
905
906
907
908
909
  private AMSchedulerEventTALaunchRequest createLaunchRequestEvent(
    TezTaskAttemptID taID, TaskAttempt ta, Resource capability,
    String[] hosts, String[] racks, Priority priority,
    ContainerContext containerContext) {
    AMSchedulerEventTALaunchRequest lr = new AMSchedulerEventTALaunchRequest(
      taID, capability, new TaskSpec(taID, "dagName", "vertexName",
      new ProcessorDescriptor("processorClassName"),
      Collections.singletonList(new InputSpec("vertexName",
        new InputDescriptor("inputClassName"), 1)),
      Collections.singletonList(new OutputSpec("vertexName",
        new OutputDescriptor("outputClassName"), 1)), null), ta, hosts, racks,
View Full Code Here

Examples of org.apache.tez.runtime.api.impl.TaskSpec

502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
              }
            }), defaultConf);
        RelocalizationUtils.addUrlsToClassPath(downloadedUrls);

        LOG.info("Done localizing additional resources");
        final TaskSpec taskSpec = containerTask.getTaskSpec();
        if (LOG.isDebugEnabled()) {
          LOG.debug("New container task context:"
              + taskSpec.toString());
        }

        try {
          taskLock.writeLock().lock();
          currentTaskAttemptID = taskSpec.getTaskAttemptID();
          TezVertexID newVertexId =
              currentTaskAttemptID.getTaskID().getVertexID();
          currentTaskComplete.set(false);

          if (lastVertexId != null) {
            if (!lastVertexId.equals(newVertexId)) {
              objectRegistry.clearCache(ObjectLifeCycle.VERTEX);
            }
            if (!lastVertexId.getDAGId().equals(newVertexId.getDAGId())) {
              objectRegistry.clearCache(ObjectLifeCycle.DAG);
              startedInputsMap = HashMultimap.create();
            }
          }
          lastVertexId = newVertexId;
          updateLoggers(currentTaskAttemptID);

          currentTask = createLogicalTask(attemptNumber, taskSpec,
              defaultConf, tezUmbilical, serviceConsumerMetadata);
         
          taskNonOobHeartbeatCounter = 0;
          nextHeartbeatNumToLog = (Math.max(1,
              (int) (LOG_COUNTER_START_INTERVAL / (amPollInterval == 0 ? 0.000001f
                  : (float) amPollInterval))));
        } finally {
          taskLock.writeLock().unlock();
        }

        final EventMetaData sourceInfo = new EventMetaData(
            EventProducerConsumerType.SYSTEM,
            taskSpec.getVertexName(), "", currentTaskAttemptID);
        currentSourceInfo = sourceInfo;

        // TODO Initiate Java VM metrics
        // JvmMetrics.initSingleton(containerId.toString(), job.getSessionId());
View Full Code Here

Examples of org.apache.tez.runtime.api.impl.TaskSpec

524
525
526
527
528
529
530
531
532
533
534
  TaskSpec createRemoteTaskSpec() {
    Vertex vertex = getVertex();
    ProcessorDescriptor procDesc = vertex.getProcessorDescriptor();
    int taskId = getTaskID().getId();
    return new TaskSpec(getID(),
        vertex.getDAG().getName(),
        vertex.getName(), procDesc,
        vertex.getInputSpecList(taskId), vertex.getOutputSpecList(taskId),
        vertex.getGroupInputSpecList(taskId));
  }
View Full Code Here

Examples of org.apache.tez.runtime.api.impl.TaskSpec

1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
      // TODO Creating the remote task here may not be required in case of
      // recovery.

      // Create the remote task.
      TaskSpec remoteTaskSpec = ta.createRemoteTaskSpec();
      // Create startTaskRequest

      String[] requestHosts = new String[0];
      String[] requestRacks = new String[0];
View Full Code Here

Examples of org.apache.tez.runtime.api.impl.TaskSpec

524
525
526
527
528
529
530
531
532
533
534
  TaskSpec createRemoteTaskSpec() {
    Vertex vertex = getVertex();
    ProcessorDescriptor procDesc = vertex.getProcessorDescriptor();
    int taskId = getTaskID().getId();
    return new TaskSpec(getID(),
        vertex.getDAG().getName(),
        vertex.getName(), procDesc,
        vertex.getInputSpecList(taskId), vertex.getOutputSpecList(taskId),
        vertex.getGroupInputSpecList(taskId));
  }
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.