Examples of TezOutputContextImpl

Examples of org.apache.tez.runtime.api.impl.TezOutputContextImpl

119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155

    OutputDescriptor outputDescriptor = mock(OutputDescriptor.class);
    when(outputDescriptor.getClassName()).thenReturn("OutputDescriptor");

    OutputContext outputContext = new TezOutputContextImpl(conf, new String[] {workDir.toString()},
        appAttemptNumber, tezUmbilical, dagName, taskVertexName, destinationVertexName,
        -1, taskAttemptID, counters, 0, userPayload, runtimeTask,
        null, auxEnv, new MemoryDistributor(1, 1, conf) , outputDescriptor, null);

    UnorderedKVOutput kvOutput = new OnFileUnorderedKVOutputForTest(outputContext, 1);

    List<Event> events = null;

    events = kvOutput.initialize();
    assertTrue(events != null && events.size() == 0);

    KeyValueWriter kvWriter = kvOutput.getWriter();
    List<KVPair> data = KVDataGen.generateTestData(true);
    for (KVPair kvp : data) {
      kvWriter.write(kvp.getKey(), kvp.getvalue());
    }

    events = kvOutput.close();
    assertTrue(events != null && events.size() == 1);
    DataMovementEvent dmEvent = (DataMovementEvent)events.get(0);

    assertEquals("Invalid source index", 0, dmEvent.getSourceIndex());

    DataMovementEventPayloadProto shufflePayload = DataMovementEventPayloadProto
        .parseFrom(ByteString.copyFrom(dmEvent.getUserPayload()));

    assertFalse(shufflePayload.hasEmptyPartitions());
    assertEquals(outputContext.getUniqueIdentifier(), shufflePayload.getPathComponent());
    assertEquals(shufflePort, shufflePayload.getPort());
    assertEquals("host", shufflePayload.getHost());
  }
View Full Code Here

Examples of org.apache.tez.runtime.api.impl.TezOutputContextImpl

471
472
473
474
475
476
477
478
479
480
481
        inputSpec.getInputDescriptor(), input, inputReadyTracker);
    return inputContext;
  }

  private TezOutputContext createOutputContext(OutputSpec outputSpec, int outputIndex) {
    TezOutputContext outputContext = new TezOutputContextImpl(tezConf,
        appAttemptNumber, tezUmbilical, taskSpec.getVertexName(),
        outputSpec.getDestinationVertexName(), taskSpec.getTaskAttemptID(),
        tezCounters, outputIndex,
        outputSpec.getOutputDescriptor().getUserPayload() == null ? taskSpec
            .getProcessorDescriptor().getUserPayload() : outputSpec
View Full Code Here

Examples of org.apache.tez.runtime.api.impl.TezOutputContextImpl

116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
    bb.putInt(shufflePort);
    bb.position(0);
    AuxiliaryServiceHelper.setServiceDataIntoEnv(ShuffleUtils.SHUFFLE_HANDLER_SERVICE_ID, bb, auxEnv);


    TezOutputContext outputContext = new TezOutputContextImpl(conf,
        appAttemptNumber, tezUmbilical, taskVertexName, destinationVertexName,
        taskAttemptID, counters, 0, userPayload, runtimeTask,
        null, auxEnv, new MemoryDistributor(1, 1, conf) , mock(OutputDescriptor.class));

    List<Event> events = null;

    events = kvOutput.initialize(outputContext);
    assertTrue(events != null && events.size() == 0);

    KeyValueWriter kvWriter = kvOutput.getWriter();
    List<KVPair> data = KVDataGen.generateTestData(true);
    for (KVPair kvp : data) {
      kvWriter.write(kvp.getKey(), kvp.getvalue());
    }

    events = kvOutput.close();
    assertTrue(events != null && events.size() == 1);
    DataMovementEvent dmEvent = (DataMovementEvent)events.get(0);

    assertEquals("Invalid source index", 0, dmEvent.getSourceIndex());

    DataMovementEventPayloadProto shufflePayload = DataMovementEventPayloadProto
        .parseFrom(dmEvent.getUserPayload());

    assertTrue(shufflePayload.getOutputGenerated());
    assertEquals(outputContext.getUniqueIdentifier(), shufflePayload.getPathComponent());
    assertEquals(shufflePort, shufflePayload.getPort());
    assertEquals("host", shufflePayload.getHost());
  }
View Full Code Here

Examples of org.apache.tez.runtime.api.impl.TezOutputContextImpl

113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
    bb.putInt(shufflePort);
    bb.position(0);
    AuxiliaryServiceHelper.setServiceDataIntoEnv(ShuffleUtils.SHUFFLE_HANDLER_SERVICE_ID, bb, auxEnv);

   
    TezOutputContext outputContext = new TezOutputContextImpl(conf,
        appAttemptNumber, tezUmbilical, taskVertexName, destinationVertexName,
        taskAttemptID, counters, userPayload, runtimeTask,
        null, auxEnv);

    List<Event> events = null;

    events = kvOutput.initialize(outputContext);
    assertTrue(events != null && events.size() == 0);

    KeyValueWriter kvWriter = kvOutput.getWriter();
    List<KVPair> data = KVDataGen.generateTestData(true);
    for (KVPair kvp : data) {
      kvWriter.write(kvp.getKey(), kvp.getvalue());
    }

    events = kvOutput.close();
    assertTrue(events != null && events.size() == 1);
    DataMovementEvent dmEvent = (DataMovementEvent)events.get(0);

    assertEquals("Invalid source index", 0, dmEvent.getSourceIndex());

    DataMovementEventPayloadProto shufflePayload = DataMovementEventPayloadProto
        .parseFrom(dmEvent.getUserPayload());

    assertTrue(shufflePayload.getOutputGenerated());
    assertEquals(outputContext.getUniqueIdentifier(), shufflePayload.getPathComponent());
    assertEquals(shufflePort, shufflePayload.getPort());
    assertEquals("host", shufflePayload.getHost());
  }
View Full Code Here

Examples of org.apache.tez.runtime.api.impl.TezOutputContextImpl

318
319
320
321
322
323
324
325
326
327
328
        serviceConsumerMetadata, System.getenv());
    return inputContext;
  }

  private TezOutputContext createOutputContext(OutputSpec outputSpec) {
    TezOutputContext outputContext = new TezOutputContextImpl(tezConf,
        appAttemptNumber, tezUmbilical, taskSpec.getVertexName(),
        outputSpec.getDestinationVertexName(), taskSpec.getTaskAttemptID(),
        tezCounters,
        outputSpec.getOutputDescriptor().getUserPayload() == null ? taskSpec
            .getProcessorDescriptor().getUserPayload() : outputSpec
View Full Code Here

Examples of org.apache.tez.runtime.api.impl.TezOutputContextImpl

488
489
490
491
492
493
494
495
496
497
498
        inputSpec.getInputDescriptor(), inputMap, inputReadyTracker, objectRegistry);
    return inputContext;
  }

  private OutputContext createOutputContext(OutputSpec outputSpec, int outputIndex) {
    OutputContext outputContext = new TezOutputContextImpl(tezConf, localDirs,
        appAttemptNumber, tezUmbilical,
        taskSpec.getDAGName(), taskSpec.getVertexName(),
        outputSpec.getDestinationVertexName(),
        taskSpec.getVertexParallelism(),
        taskSpec.getTaskAttemptID(),
View Full Code Here

Examples of org.apache.tez.runtime.api.impl.TezOutputContextImpl

118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
    bb.putInt(shufflePort);
    bb.position(0);
    AuxiliaryServiceHelper.setServiceDataIntoEnv(ShuffleUtils.SHUFFLE_HANDLER_SERVICE_ID, bb, auxEnv);


    TezOutputContext outputContext = new TezOutputContextImpl(conf,
        appAttemptNumber, tezUmbilical, dagName, taskVertexName, destinationVertexName,
        taskAttemptID, counters, 0, userPayload, runtimeTask,
        null, auxEnv, new MemoryDistributor(1, 1, conf) , mock(OutputDescriptor.class));

    List<Event> events = null;

    events = kvOutput.initialize(outputContext);
    assertTrue(events != null && events.size() == 0);

    KeyValueWriter kvWriter = kvOutput.getWriter();
    List<KVPair> data = KVDataGen.generateTestData(true);
    for (KVPair kvp : data) {
      kvWriter.write(kvp.getKey(), kvp.getvalue());
    }

    events = kvOutput.close();
    assertTrue(events != null && events.size() == 1);
    DataMovementEvent dmEvent = (DataMovementEvent)events.get(0);

    assertEquals("Invalid source index", 0, dmEvent.getSourceIndex());

    DataMovementEventPayloadProto shufflePayload = DataMovementEventPayloadProto
        .parseFrom(dmEvent.getUserPayload());

    assertFalse(shufflePayload.hasEmptyPartitions());
    assertEquals(outputContext.getUniqueIdentifier(), shufflePayload.getPathComponent());
    assertEquals(shufflePort, shufflePayload.getPort());
    assertEquals("host", shufflePayload.getHost());
  }
View Full Code Here

Examples of org.apache.tez.runtime.api.impl.TezOutputContextImpl

473
474
475
476
477
478
479
480
481
482
483
        inputSpec.getInputDescriptor(), input, inputReadyTracker);
    return inputContext;
  }

  private TezOutputContext createOutputContext(OutputSpec outputSpec, int outputIndex) {
    TezOutputContext outputContext = new TezOutputContextImpl(tezConf,
        appAttemptNumber, tezUmbilical,
        taskSpec.getDAGName(), taskSpec.getVertexName(),
        outputSpec.getDestinationVertexName(), taskSpec.getTaskAttemptID(),
        tezCounters, outputIndex,
        outputSpec.getOutputDescriptor().getUserPayload() == null ? taskSpec
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.