Package org.apache.tez.dag.api

Examples of org.apache.tez.dag.api.TezUncheckedException

842
843
844
845
846
847
848
849
   
    UnionExample job = new UnionExample();
    if (job.run(inPath.toString(), outPath.toString(), mrrTezCluster.getConfig())) {
      LOG.info("Success VertexGroups Test");
    } else {
      throw new TezUncheckedException("VertexGroups Test Failed");
    }
  }
View Full Code Here
853
854
855
856
857
858
859
860
    LOG.info("Running BroadcastAndOneToOne Test");
    BroadcastAndOneToOneExample job = new BroadcastAndOneToOneExample();
    if (job.run(mrrTezCluster.getConfig(), true)) {
      LOG.info("Success BroadcastAndOneToOne Test");
    } else {
      throw new TezUncheckedException("BroadcastAndOneToOne Test Failed");
    }
  }
View Full Code Here
349
350
351
352
353
354
355
356
357
358
359
      // save output size
      VertexManagerEventPayloadProto proto;
      try {
        proto = VertexManagerEventPayloadProto.parseFrom(ByteString.copyFrom(vmEvent.getUserPayload()));
      } catch (InvalidProtocolBufferException e) {
        throw new TezUncheckedException(e);
      }
      long sourceTaskOutputSize = proto.getOutputSize();
      numVertexManagerEventsReceived++;
      completedSourceTasksOutputSize += sourceTaskOutputSize;
      if (LOG.isDebugEnabled()) {
View Full Code Here
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
  public void initialize() {
    Configuration conf;
    try {
      conf = TezUtils.createConfFromUserPayload(getContext().getUserPayload());
    } catch (IOException e) {
      throw new TezUncheckedException(e);
    }

    this.slowStartMinSrcCompletionFraction = conf
        .getFloat(
            ShuffleVertexManager.TEZ_SHUFFLE_VERTEX_MANAGER_MIN_SRC_FRACTION,
            ShuffleVertexManager.TEZ_SHUFFLE_VERTEX_MANAGER_MIN_SRC_FRACTION_DEFAULT);
    this.slowStartMaxSrcCompletionFraction = conf
        .getFloat(
            ShuffleVertexManager.TEZ_SHUFFLE_VERTEX_MANAGER_MAX_SRC_FRACTION,
            ShuffleVertexManager.TEZ_SHUFFLE_VERTEX_MANAGER_MAX_SRC_FRACTION_DEFAULT);

    if (slowStartMinSrcCompletionFraction < 0
        || slowStartMaxSrcCompletionFraction < slowStartMinSrcCompletionFraction) {
      throw new IllegalArgumentException(
          "Invalid values for slowStartMinSrcCompletionFraction"
              + "/slowStartMaxSrcCompletionFraction. Min cannot be < 0 and "
              + "max cannot be < min.");
    }

    enableAutoParallelism = conf
        .getBoolean(
            ShuffleVertexManager.TEZ_SHUFFLE_VERTEX_MANAGER_ENABLE_AUTO_PARALLEL,
            ShuffleVertexManager.TEZ_SHUFFLE_VERTEX_MANAGER_ENABLE_AUTO_PARALLEL_DEFAULT);
    desiredTaskInputDataSize = conf
        .getLong(
            ShuffleVertexManager.TEZ_SHUFFLE_VERTEX_MANAGER_DESIRED_TASK_INPUT_SIZE,
            ShuffleVertexManager.TEZ_SHUFFLE_VERTEX_MANAGER_DESIRED_TASK_INPUT_SIZE_DEFAULT);
    minTaskParallelism = Math.max(1, conf
        .getInt(
            ShuffleVertexManager.TEZ_SHUFFLE_VERTEX_MANAGER_MIN_TASK_PARALLELISM,
            ShuffleVertexManager.TEZ_SHUFFLE_VERTEX_MANAGER_MIN_TASK_PARALLELISM_DEFAULT));
    LOG.info("Shuffle Vertex Manager: settings" + " minFrac:"
        + slowStartMinSrcCompletionFraction + " maxFrac:"
        + slowStartMaxSrcCompletionFraction + " auto:" + enableAutoParallelism
        + " desiredTaskIput:" + desiredTaskInputDataSize + " minTasks:"
        + minTaskParallelism);
   
    Map<String, EdgeProperty> inputs = getContext().getInputVertexEdgeProperties();
    for(Map.Entry<String, EdgeProperty> entry : inputs.entrySet()) {
      if (entry.getValue().getDataMovementType() == DataMovementType.SCATTER_GATHER) {
        String vertex = entry.getKey();
        bipartiteSources.put(vertex, new HashSet<Integer>());
      }
    }
    if(bipartiteSources.isEmpty()) {
      throw new TezUncheckedException("Atleast 1 bipartite source should exist");
    }
    // dont track the source tasks here since those tasks may themselves be
    // dynamically changed as the DAG progresses.

  }
View Full Code Here
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
      TezEvent tezEvent = outputFailedEvent.getInputFailedEvent();
      TezTaskAttemptID failedDestTaId = tezEvent.getSourceInfo().getTaskAttemptID();
      InputReadErrorEvent readErrorEvent = (InputReadErrorEvent)tezEvent.getEvent();
      int failedInputIndexOnDestTa = readErrorEvent.getIndex();
      if (readErrorEvent.getVersion() != attempt.getID().getId()) {
        throw new TezUncheckedException(attempt.getID()
            + " incorrectly blamed for read error from " + failedDestTaId
            + " at inputIndex " + failedInputIndexOnDestTa + " version"
            + readErrorEvent.getVersion());
      }
      LOG.info(attempt.getID()
View Full Code Here
76
77
78
79
80
81
82
83
84
85
86
      if(vertex != currentPartitioner) {
        String message = vertex.getVertexId() + " finished. Expecting"
            + " current partitioner " + currentPartitioner.getVertexId()
            + " to finish.";
        LOG.fatal(message);
        throw new TezUncheckedException(message);
      }
      LOG.info("Current partitioner " + currentPartitioner.getVertexId()
          + " is completed. "
          + (currentShuffler!=null ?
             currentShuffler.getVertexId() + " is new partitioner":
View Full Code Here
151
152
153
154
155
156
157
158
159
160
       vertexDistanceFromRoot >= currentPartitioner.getDistanceFromRoot()) {
      String message = vertex.getVertexId() + " is neither the "
          + " current partitioner: " + currentPartitioner.getVertexId()
          + " nor the current shuffler: " + currentShuffler.getVertexId();
      LOG.fatal(message);
      throw new TezUncheckedException(message);     
    }
   
    scheduleTaskAttempt(attempt);
  }
View Full Code Here
663
664
665
666
667
668
669
670
          VertexManagerPluginDescriptor.create(ShuffleVertexManager.class.getName());

      try {
        return desc.setUserPayload(TezUtils.createUserPayloadFromConf(this.conf));
      } catch (IOException e) {
        throw new TezUncheckedException(e);
      }
    }
View Full Code Here
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
      EdgeProperty edgeProp = entry.getValue();
      String srcVertex = entry.getKey();
      int numSrcTasks = getContext().getVertexNumTasks(srcVertex);
      switch (edgeProp.getDataMovementType()) {
      case CUSTOM:
        throw new TezUncheckedException("Cannot handle custom edge");
      case ONE_TO_ONE:
        numOneToOneEdges++;
        if (oneToOneSrcTaskCount == 0) {
          oneToOneSrcTaskCount = numSrcTasks;
        } else if (oneToOneSrcTaskCount != numSrcTasks) {
          throw new TezUncheckedException(
              "All 1-1 source vertices must have identical concurrency");
        }
        break;
      case SCATTER_GATHER:
      case BROADCAST:
        break;
      default:
        throw new TezUncheckedException(
            "Unknown edge type: " + edgeProp.getDataMovementType());
      }
      srcVertexInfo.put(srcVertex, new SourceVertexInfo(numSrcTasks, edgeProp));
    }
   
    if (numOneToOneEdges > 0) {
      if (oneToOneSrcTaskCount != numManagedTasks) {
        throw new TezUncheckedException(
            "Managed task number must equal 1-1 source task number");
      }
      oneToOneSrcTasksDoneCount = new int[oneToOneSrcTaskCount];
      oneToOneLocationHints = new Container[oneToOneSrcTaskCount];
    }
View Full Code Here
125
126
127
128
129
130
131
132
133
134
135
      // Add shuffle token
      LOG.info("Putting shuffle token in serviceData");
      serviceData.put(TezConstants.TEZ_SHUFFLE_HANDLER_SERVICE_ID,
          serializeServiceData(TokenCache.getSessionToken(containerCredentials)));
    } catch (IOException e) {
      throw new TezUncheckedException(e);
    }
    // Construct the actual Container
    // The null fields are per-container and will be constructed for each
    // container separately.
    ContainerLaunchContext container =
View Full Code Here
TOP

Related Classes of org.apache.tez.dag.api.TezUncheckedException

  • org.apache.hadoop.mapred.MRVertexOutputCommitter
  • org.apache.hadoop.mapred.split.TezGroupedSplit
  • org.apache.hadoop.mapred.split.TezGroupedSplitsInputFormat
  • org.apache.hadoop.mapred.split.TezMapredSplitsGrouper
  • org.apache.hadoop.mapreduce.split.TezGroupedSplit
  • org.apache.hadoop.mapreduce.split.TezGroupedSplitsInputFormat
  • org.apache.hadoop.mapreduce.split.TezGroupedSplitsInputFormat$TezGroupedSplitsRecordReader
  • org.apache.hadoop.mapreduce.split.TezMapReduceSplitsGrouper
  • org.apache.tez.client.AMConfiguration
  • org.apache.tez.client.TezClientUtils
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.