Examples of MRSplitProto

Examples of org.apache.tez.mapreduce.protos.MRRuntimeProtos.MRSplitProto

103
104
105
106
107
108
109
110
111
112
113
114
    Configuration conf = new Configuration(false);
    conf.setBoolean(MRJobConfig.MR_TEZ_INPUT_INITIALIZER_SERIALIZE_EVENT_PAYLAOD, false);
    ByteString confByteString = MRHelpers.createByteStringFromConf(conf);
    InputSplit split1 = new InputSplitForTest(1);
    InputSplit split2 = new InputSplitForTest(2);
    MRSplitProto proto1 = MRHelpers.createSplitProto(split1);
    MRSplitProto proto2 = MRHelpers.createSplitProto(split2);
    MRSplitsProto.Builder splitsProtoBuilder = MRSplitsProto.newBuilder();
    splitsProtoBuilder.addSplits(proto1);
    splitsProtoBuilder.addSplits(proto2);
    MRInputUserPayloadProto.Builder payloadProto = MRInputUserPayloadProto.newBuilder();
    payloadProto.setSplits(splitsProtoBuilder.build());
View Full Code Here

Examples of org.apache.tez.mapreduce.protos.MRRuntimeProtos.MRSplitProto

368
369
370
371
372
373
374
375
376
377
378
 
  private void initFromEventInternal(RootInputDataInformationEvent initEvent)
      throws IOException {
    LOG.info("Initializing RecordReader from event");
    Preconditions.checkState(initEvent != null, "InitEvent must be specified");
    MRSplitProto splitProto = MRSplitProto
        .parseFrom(initEvent.getUserPayload());
    if (useNewApi) {
      newInputSplit = getNewSplitDetailsFromEvent(splitProto, jobConf);
      LOG.info("Split Details -> SplitClass: "
          + newInputSplit.getClass().getName() + ", NewSplit: " + newInputSplit);
View Full Code Here

Examples of org.apache.tez.mapreduce.protos.MRRuntimeProtos.MRSplitProto

362
363
364
365
366
367
368
369
370
371
372
 
  private void initFromEventInternal(RootInputDataInformationEvent initEvent)
      throws IOException {
    LOG.info("Initializing RecordReader from event");
    Preconditions.checkState(initEvent != null, "InitEvent must be specified");
    MRSplitProto splitProto = MRSplitProto
        .parseFrom(initEvent.getUserPayload());
    if (useNewApi) {
      newInputSplit = getNewSplitDetailsFromEvent(splitProto);
      LOG.info("Split Details -> SplitClass: "
          + newInputSplit.getClass().getName() + ", NewSplit: " + newInputSplit);
View Full Code Here

Examples of org.apache.tez.mapreduce.protos.MRRuntimeProtos.MRSplitProto

55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
    Configuration conf = new Configuration(false);
    conf.setBoolean(MRJobConfig.MR_TEZ_INPUT_INITIALIZER_SERIALIZE_EVENT_PAYLAOD, true);
    ByteString confByteString = MRHelpers.createByteStringFromConf(conf);
    InputSplit split1 = new InputSplitForTest(1);
    InputSplit split2 = new InputSplitForTest(2);
    MRSplitProto proto1 = MRHelpers.createSplitProto(split1);
    MRSplitProto proto2 = MRHelpers.createSplitProto(split2);
    MRSplitsProto.Builder splitsProtoBuilder = MRSplitsProto.newBuilder();
    splitsProtoBuilder.addSplits(proto1);
    splitsProtoBuilder.addSplits(proto2);
    MRInputUserPayloadProto.Builder payloadProto = MRInputUserPayloadProto.newBuilder();
    payloadProto.setSplits(splitsProtoBuilder.build());
    payloadProto.setConfigurationBytes(confByteString);
    byte[] userPayload = payloadProto.build().toByteArray();

    TezRootInputInitializerContext context = new TezRootInputInitializerContextForTest(userPayload);
    MRInputSplitDistributor splitDist = new MRInputSplitDistributor();

    List<Event> events = splitDist.initialize(context);

    assertEquals(3, events.size());
    assertTrue(events.get(0) instanceof RootInputUpdatePayloadEvent);
    assertTrue(events.get(1) instanceof RootInputDataInformationEvent);
    assertTrue(events.get(2) instanceof RootInputDataInformationEvent);

    RootInputDataInformationEvent diEvent1 = (RootInputDataInformationEvent) (events.get(1));
    RootInputDataInformationEvent diEvent2 = (RootInputDataInformationEvent) (events.get(2));

    assertNull(diEvent1.getDeserializedUserPayload());
    assertNull(diEvent2.getDeserializedUserPayload());

    assertNotNull(diEvent1.getUserPayload());
    assertNotNull(diEvent2.getUserPayload());

    MRSplitProto event1Proto = MRSplitProto.parseFrom(diEvent1.getUserPayload());
    InputSplit is1 = MRInput.getOldSplitDetailsFromEvent(event1Proto, new Configuration());
    assertTrue(is1 instanceof InputSplitForTest);
    assertEquals(1, ((InputSplitForTest) is1).identifier);

    MRSplitProto event2Proto = MRSplitProto.parseFrom(diEvent2.getUserPayload());
    InputSplit is2 = MRInput.getOldSplitDetailsFromEvent(event2Proto, new Configuration());
    assertTrue(is2 instanceof InputSplitForTest);
    assertEquals(2, ((InputSplitForTest) is2).identifier);
  }
View Full Code Here

Examples of org.apache.tez.mapreduce.protos.MRRuntimeProtos.MRSplitProto

103
104
105
106
107
108
109
110
111
112
113
114
    Configuration conf = new Configuration(false);
    conf.setBoolean(MRJobConfig.MR_TEZ_INPUT_INITIALIZER_SERIALIZE_EVENT_PAYLAOD, false);
    ByteString confByteString = MRHelpers.createByteStringFromConf(conf);
    InputSplit split1 = new InputSplitForTest(1);
    InputSplit split2 = new InputSplitForTest(2);
    MRSplitProto proto1 = MRHelpers.createSplitProto(split1);
    MRSplitProto proto2 = MRHelpers.createSplitProto(split2);
    MRSplitsProto.Builder splitsProtoBuilder = MRSplitsProto.newBuilder();
    splitsProtoBuilder.addSplits(proto1);
    splitsProtoBuilder.addSplits(proto2);
    MRInputUserPayloadProto.Builder payloadProto = MRInputUserPayloadProto.newBuilder();
    payloadProto.setSplits(splitsProtoBuilder.build());
View Full Code Here

Examples of org.apache.tez.mapreduce.protos.MRRuntimeProtos.MRSplitProto

346
347
348
349
350
351
352
353
354
355
356
 
  private void initFromEventInternal(RootInputDataInformationEvent initEvent)
      throws IOException {
    LOG.info("Initializing RecordReader from event");
    Preconditions.checkState(initEvent != null, "InitEvent must be specified");
    MRSplitProto splitProto = MRSplitProto
        .parseFrom(initEvent.getUserPayload());
    if (useNewApi) {
      newInputSplit = getNewSplitDetailsFromEvent(splitProto, jobConf);
      LOG.info("Split Details -> SplitClass: "
          + newInputSplit.getClass().getName() + ", NewSplit: " + newInputSplit);
View Full Code Here

Examples of org.apache.tez.mapreduce.protos.MRRuntimeProtos.MRSplitProto

248
249
250
251
252
253
254
255
256
257
258
259
260
    List<RootInputDataInformationEvent> taskEvents = Lists.newArrayListWithCapacity(finalSplits.size());
    // Re-serialize the splits after grouping.
    int count = 0;
    for (InputSplit inputSplit : finalSplits) {
      MRSplitProto serializedSplit = MRHelpers.createSplitProto(inputSplit);
      RootInputDataInformationEvent diEvent = new RootInputDataInformationEvent(
          count, serializedSplit.toByteArray());
      diEvent.setTargetIndex(count);
      count++;
      taskEvents.add(diEvent);
    }
View Full Code Here

Examples of org.apache.tez.mapreduce.protos.MRRuntimeProtos.MRSplitProto

282
283
284
285
286
287
288
289
290
291
292
      throws IOException {
    InputSplit inputSplit = null;
    if (event.getDeserializedUserPayload() != null) {
      inputSplit = (InputSplit) event.getDeserializedUserPayload();
    } else {
      MRSplitProto splitProto = MRSplitProto.parseFrom(event.getUserPayload());
      SerializationFactory serializationFactory = new SerializationFactory(
          new Configuration());
      inputSplit = MRHelpers.createOldFormatSplitFromUserPayload(splitProto,
          serializationFactory);
    }
View Full Code Here

Examples of org.apache.tez.mapreduce.protos.MRRuntimeProtos.MRSplitProto

342
343
344
345
346
347
348
349
350
351
352
353
    Multimap<Integer, ByteBuffer> bucketToSerializedSplitMap = LinkedListMultimap.create();

    // Create the list of serialized splits for each bucket.
    for (Entry<Integer, Collection<InputSplit>> entry : bucketToGroupedSplitMap.asMap().entrySet()) {
      for (InputSplit split : entry.getValue()) {
        MRSplitProto serializedSplit = MRInputHelpers.createSplitProto(split);
        ByteBuffer bs = serializedSplit.toByteString().asReadOnlyByteBuffer();
        bucketToSerializedSplitMap.put(entry.getKey(), bs);
      }
    }

    for (Entry<Integer, Collection<ByteBuffer>> entry : bucketToSerializedSplitMap.asMap().entrySet()) {
View Full Code Here

Examples of org.apache.tez.mapreduce.protos.MRRuntimeProtos.MRSplitProto

430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
        for (InputSplit subSplit : tezGroupedSplit.getGroupedSplits()) {
          if ((subSplit instanceof TezGroupedSplit) == false) {
            throw new IOException("Unexpected split type found: "
                + subSplit.getClass().getCanonicalName());
          }
          MRSplitProto serializedSplit = MRInputHelpers.createSplitProto(subSplit);
          InputDataInformationEvent diEvent =
              InputDataInformationEvent.createWithSerializedPayload(count, serializedSplit
                  .toByteString().asReadOnlyByteBuffer());
          diEvent.setTargetIndex(count);
          taskEvents.add(diEvent);
        }
      } else {
        MRSplitProto serializedSplit = MRInputHelpers.createSplitProto(inputSplit);
        InputDataInformationEvent diEvent =
            InputDataInformationEvent.createWithSerializedPayload(count, serializedSplit
                .toByteString().asReadOnlyByteBuffer());
        diEvent.setTargetIndex(count);
        taskEvents.add(diEvent);
      }
      count++;
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.