Package org.apache.tez.runtime.library.common.shuffle.impl

Examples of org.apache.tez.runtime.library.common.shuffle.impl.Shuffle

117
118
119
120
121
122
123
124
125
126
127
  @Override
  public synchronized void start() throws IOException {
    if (!isStarted.get()) {
      memoryUpdateCallbackHandler.validateUpdateReceived();
      // Start the shuffle - copy and merge
      shuffle = new Shuffle(getContext(), conf, getNumPhysicalInputs(), memoryUpdateCallbackHandler.getMemoryAssigned());
      shuffle.run();
      if (LOG.isDebugEnabled()) {
        LOG.debug("Initialized the handlers in shuffle..Safe to start processing..");
      }
      List<Event> pending = new LinkedList<Event>();
View Full Code Here
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
   * @throws IOException
   * @throws InterruptedException
   */
  public void waitForInputReady() throws IOException, InterruptedException, TezException {
    // Cannot synchronize entire method since this is called form user code and can block.
    Shuffle localShuffleCopy = null;
    synchronized (this) {
      Preconditions.checkState(isStarted.get(), "Must start input before invoking this method");
      if (getNumPhysicalInputs() == 0) {
        return;
      }
      localShuffleCopy = shuffle;
    }

    TezRawKeyValueIterator localRawIter = localShuffleCopy.waitForInput();
    synchronized(this) {
      rawIter = localRawIter;
      createValuesIterator();
    }
  }
View Full Code Here
86
87
88
89
90
91
92
93
    this.inputKeyCounter = inputContext.getCounters().findCounter(TaskCounter.REDUCE_INPUT_GROUPS);
    this.inputValueCounter = inputContext.getCounters().findCounter(
        TaskCounter.REDUCE_INPUT_RECORDS);
    this.conf.setStrings(TezJobConfig.LOCAL_DIRS, inputContext.getWorkDirs());

    shuffle = new Shuffle(inputContext, this.conf, numInputs);
    return Collections.emptyList();
  }
View Full Code Here
77
78
79
80
81
82
83
84
85
86
    this.inputValueCounter = inputContext.getCounters().findCounter(TaskCounter.REDUCE_INPUT_RECORDS);
    this.conf.setStrings(TezJobConfig.LOCAL_DIRS,
        inputContext.getWorkDirs());

    // Start the shuffle - copy and merge.
    shuffle = new Shuffle(inputContext, this.conf, numInputs);
    shuffle.run();

    return Collections.emptyList();
  }
View Full Code Here
107
108
109
110
111
112
113
114
115
116
117
  @Override
  public synchronized void start() throws IOException {
    if (!isStarted.get()) {
      memoryUpdateCallbackHandler.validateUpdateReceived();
      // Start the shuffle - copy and merge
      shuffle = new Shuffle(inputContext, conf, numInputs, memoryUpdateCallbackHandler.getMemoryAssigned());
      shuffle.run();
      if (LOG.isDebugEnabled()) {
        LOG.debug("Initialized the handlers in shuffle..Safe to start processing..");
      }
      List<Event> pending = new LinkedList<Event>();
View Full Code Here
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
   * @throws IOException
   * @throws InterruptedException
   */
  public void waitForInputReady() throws IOException, InterruptedException {
    // Cannot synchronize entire method since this is called form user code and can block.
    Shuffle localShuffleCopy = null;
    synchronized (this) {
      Preconditions.checkState(isStarted.get(), "Must start input before invoking this method");
      if (this.numInputs == 0) {
        return;
      }
      localShuffleCopy = shuffle;
    }
   
    TezRawKeyValueIterator localRawIter = localShuffleCopy.waitForInput();
    synchronized(this) {
      rawIter = localRawIter;
      createValuesIterator();
    }
  }
View Full Code Here
TOP

Related Classes of org.apache.tez.runtime.library.common.shuffle.impl.Shuffle

  • com.google.common.util.concurrent.ThreadFactoryBuilder
  • org.apache.hadoop.fs.FileSystem
  • org.apache.hadoop.fs.LocalDirAllocator
  • org.apache.tez.common.counters.TezCounter
  • org.apache.tez.runtime.library.common.combine.Combiner
  • org.apache.tez.runtime.library.common.sort.impl.TezRawKeyValueIterator
  • org.apache.tez.runtime.library.input.OrderedGroupedKVInput
  • org.apache.tez.runtime.library.input.ShuffledMergedInput
  • java.util.concurrent.ExecutorService
  • java.lang.reflect.UndeclaredThrowableException
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.