Examples of TezInputContext

Examples of org.apache.tez.runtime.api.TezInputContext

246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
    @Override
    public Void call() throws Exception {
      LOG.info("Initializing Input using InputSpec: " + inputSpec);
      String edgeName = inputSpec.getSourceVertexName();
      LogicalInput input = createInput(inputSpec);
      TezInputContext inputContext = createInputContext(inputSpec);
      inputsMap.put(edgeName, input);
      inputContextMap.put(edgeName, inputContext);

      if (input instanceof LogicalInput) {
        ((LogicalInput) input).setNumPhysicalInputs(inputSpec
            .getPhysicalEdgeCount());
      }
      LOG.info("Initializing Input with src edge: " + edgeName);
      List<Event> events = input.initialize(inputContext);
      sendTaskGeneratedEvents(events, EventProducerConsumerType.INPUT,
          inputContext.getTaskVertexName(), inputContext.getSourceVertexName(),
          taskSpec.getTaskAttemptID());
      LOG.info("Initialized Input with src edge: " + edgeName);
      return null;
    }
View Full Code Here

Examples of org.apache.tez.runtime.api.TezInputContext

62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
    dist.setJvmMemory(10000l);

    // First request - ScatterGatherShuffleInput
    MemoryUpdateCallbackForTest e1Callback = new MemoryUpdateCallbackForTest();
    TezInputContext e1InputContext1 = createTestInputContext();
    InputDescriptor e1InDesc1 = createTestInputDescriptor(ShuffledMergedInput.class);
    dist.requestMemory(10000, e1Callback, e1InputContext1, e1InDesc1);

    // Second request - BroadcastInput
    MemoryUpdateCallbackForTest e2Callback = new MemoryUpdateCallbackForTest();
    TezInputContext e2InputContext2 = createTestInputContext();
    InputDescriptor e2InDesc2 = createTestInputDescriptor(ShuffledUnorderedKVInput.class);
    dist.requestMemory(10000, e2Callback, e2InputContext2, e2InDesc2);

    // Third request - randomOutput (simulates MROutput)
    MemoryUpdateCallbackForTest e3Callback = new MemoryUpdateCallbackForTest();
View Full Code Here

Examples of org.apache.tez.runtime.api.TezInputContext

109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
    dist.setJvmMemory(10000l);

    // First request - ScatterGatherShuffleInput [weight 6]
    MemoryUpdateCallbackForTest e1Callback = new MemoryUpdateCallbackForTest();
    TezInputContext e1InputContext1 = createTestInputContext();
    InputDescriptor e1InDesc1 = createTestInputDescriptor(ShuffledMergedInput.class);
    dist.requestMemory(10000, e1Callback, e1InputContext1, e1InDesc1);

    // Second request - BroadcastInput [weight 2]
    MemoryUpdateCallbackForTest e2Callback = new MemoryUpdateCallbackForTest();
    TezInputContext e2InputContext2 = createTestInputContext();
    InputDescriptor e2InDesc2 = createTestInputDescriptor(ShuffledUnorderedKVInput.class);
    dist.requestMemory(10000, e2Callback, e2InputContext2, e2InDesc2);

    // Third request - randomOutput (simulates MROutput) [weight 1]
    MemoryUpdateCallbackForTest e3Callback = new MemoryUpdateCallbackForTest();
View Full Code Here

Examples of org.apache.tez.runtime.api.TezInputContext

136
137
138
139
140
141
142
143
144
145
146
147
   
    task.initialize();
    task.run();
    task.close();
   
    TezInputContext inputContext = task.getInputContexts().iterator().next();
    TezTaskOutput mapOutputs = new TezLocalTaskOutputFiles(jobConf, inputContext.getUniqueIdentifier());
   
   
    // TODO NEWTEZ FIXME OutputCommitter verification
//    MRTask mrTask = (MRTask)t.getProcessor();
//    Assert.assertEquals(TezNullOutputCommitter.class.getName(), mrTask
View Full Code Here

Examples of org.apache.tez.runtime.api.TezInputContext

459
460
461
462
463
464
465
466
467
468
469
    LOG.info("Initialized processor" + ", processorClassName="
        + processorDescriptor.getClassName());
  }

  private TezInputContext createInputContext(Input input, InputSpec inputSpec, int inputIndex) {
    TezInputContext inputContext = new TezInputContextImpl(tezConf,
        appAttemptNumber, tezUmbilical,
        taskSpec.getDAGName(), taskSpec.getVertexName(),
        inputSpec.getSourceVertexName(), taskSpec.getTaskAttemptID(),
        tezCounters, inputIndex,
        inputSpec.getInputDescriptor().getUserPayload() == null ? taskSpec
View Full Code Here

Examples of org.apache.tez.runtime.api.TezInputContext

354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
    @Override
    public Void call() throws Exception {
      LOG.info("Initializing Input using InputSpec: " + inputSpec);
      String edgeName = inputSpec.getSourceVertexName();
      LogicalInput input = createInput(inputSpec);
      TezInputContext inputContext = createInputContext(input, inputSpec, inputIndex);
      inputsMap.put(edgeName, input);
      inputContextMap.put(edgeName, inputContext);

      if (input instanceof LogicalInput) {
        ((LogicalInput) input).setNumPhysicalInputs(inputSpec
            .getPhysicalEdgeCount());
      }
      LOG.info("Initializing Input with src edge: " + edgeName);
      List<Event> events = input.initialize(inputContext);
      sendTaskGeneratedEvents(events, EventProducerConsumerType.INPUT,
          inputContext.getTaskVertexName(), inputContext.getSourceVertexName(),
          taskSpec.getTaskAttemptID());
      LOG.info("Initialized Input with src edge: " + edgeName);
      return null;
    }
View Full Code Here

Examples of org.apache.tez.runtime.api.TezInputContext

44
45
46
47
48
49
50
51
52
53
54
  private static final int PORT = 8080;
  private static final String PATH_COMPONENT = "attempttmp";

  @Test
  public void testSimple() throws IOException {
    TezInputContext inputContext = mock(TezInputContext.class);
    ShuffleManager shuffleManager = mock(ShuffleManager.class);
    FetchedInputAllocator inputAllocator = mock(FetchedInputAllocator.class);

    ShuffleInputEventHandlerImpl handler = new ShuffleInputEventHandlerImpl(inputContext,
        shuffleManager, inputAllocator, null, false, 0);
View Full Code Here

Examples of org.apache.tez.runtime.api.TezInputContext

66
67
68
69
70
71
72
73
74
75
76
    verify(shuffleManager).addKnownInput(eq(HOST), eq(PORT), eq(expectedIdentifier), eq(0));
  }

  @Test
  public void testCurrentPartitionEmpty() throws IOException {
    TezInputContext inputContext = mock(TezInputContext.class);
    ShuffleManager shuffleManager = mock(ShuffleManager.class);
    FetchedInputAllocator inputAllocator = mock(FetchedInputAllocator.class);

    ShuffleInputEventHandlerImpl handler = new ShuffleInputEventHandlerImpl(inputContext,
        shuffleManager, inputAllocator, null, false, 0);
View Full Code Here

Examples of org.apache.tez.runtime.api.TezInputContext

87
88
89
90
91
92
93
94
95
96
97
    verify(shuffleManager).addCompletedInputWithNoData(eq(expectedIdentifier));
  }

  @Test
  public void testOtherPartitionEmpty() throws IOException {
    TezInputContext inputContext = mock(TezInputContext.class);
    ShuffleManager shuffleManager = mock(ShuffleManager.class);
    FetchedInputAllocator inputAllocator = mock(FetchedInputAllocator.class);

    ShuffleInputEventHandlerImpl handler = new ShuffleInputEventHandlerImpl(inputContext,
        shuffleManager, inputAllocator, null, false, 0);
View Full Code Here

Examples of org.apache.tez.runtime.api.TezInputContext

107
108
109
110
111
112
113
114
115
116
117
    verify(shuffleManager).addKnownInput(eq(HOST), eq(PORT), eq(expectedIdentifier), eq(0));
  }

  @Test
  public void testMultipleEvents1() throws IOException {
    TezInputContext inputContext = mock(TezInputContext.class);
    ShuffleManager shuffleManager = mock(ShuffleManager.class);
    FetchedInputAllocator inputAllocator = mock(FetchedInputAllocator.class);

    ShuffleInputEventHandlerImpl handler = new ShuffleInputEventHandlerImpl(inputContext,
        shuffleManager, inputAllocator, null, false, 0);
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.