Package org.apache.tez.runtime.library.common

Examples of org.apache.tez.runtime.library.common.InputAttemptIdentifier

120
121
122
123
124
125
126
127
128
    Event dme = createDataMovementEvent(srcIdx, targetIdx, createEmptyPartitionByteString(srcIdx)
        , true);
    events.add(dme);
    handler.handleEvents(events);

    InputAttemptIdentifier expectedIdentifier = new InputAttemptIdentifier(targetIdx, 0);
    verify(scheduler).copySucceeded(eq(expectedIdentifier), any(MapHost.class), eq(0l),
        eq(0l), eq(0l), any(MapOutput.class));
  }
View Full Code Here
135
136
137
138
139
140
141
142
143
144
    Event dme = createDataMovementEvent(srcIdx, targetIdx, createEmptyPartitionByteString(srcIdx)
        , false);
    events.add(dme);
    handler.handleEvents(events);

    InputAttemptIdentifier expectedIdentifier = new InputAttemptIdentifier(targetIdx, 0);

    verify(scheduler).copySucceeded(eq(expectedIdentifier), any(MapHost.class), eq(0l),
        eq(0l), eq(0l), any(MapOutput.class));
  }
View Full Code Here
153
154
155
156
157
158
159
160
161
162
163
    events.add(dme);
    handler.handleEvents(events);

    String baseUri = handler.getBaseURI(HOST, PORT, srcIdx).toString();
    int partitionId = srcIdx;
    InputAttemptIdentifier expectedIdentifier =
        new InputAttemptIdentifier(taskIndex, 0, PATH_COMPONENT);

    verify(scheduler).addKnownMapOutput(eq(HOST), eq(PORT), eq(partitionId), eq(baseUri),
        eq(expectedIdentifier));
  }
View Full Code Here
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
    Fetcher spyFetcher = spy(fetcher);

    MapHost host = new MapHost(1, HOST + ":" + PORT,
        "http://" + HOST + ":" +PORT + "/mapOutput?job=job_123&&reduce=1&map=");
    List<InputAttemptIdentifier> srcAttempts = Arrays.asList(
        new InputAttemptIdentifier(0, 1, InputAttemptIdentifier.PATH_PREFIX + "pathComponent_0"),
        new InputAttemptIdentifier(1, 2, InputAttemptIdentifier.PATH_PREFIX + "pathComponent_1"),
        new InputAttemptIdentifier(2, 3, InputAttemptIdentifier.PATH_PREFIX + "pathComponent_2"),
        new InputAttemptIdentifier(3, 4, InputAttemptIdentifier.PATH_PREFIX + "pathComponent_3"),
        new InputAttemptIdentifier(4, 4, InputAttemptIdentifier.PATH_PREFIX + "pathComponent_4")
    );
    final int FIRST_FAILED_ATTEMPT_IDX = 2;
    final int SECOND_FAILED_ATTEMPT_IDX = 4;
    final int[] sucessfulAttemptsIndexes = {0, 1, 3};
View Full Code Here
145
146
147
148
149
150
151
152
153
154
155
156
  private void verifyCopySucceeded(ShuffleScheduler scheduler, MapHost host,
                                   List<InputAttemptIdentifier> srcAttempts, long p) throws
      IOException {
    // need to verify filename, offsets, sizes wherever they are used.
    InputAttemptIdentifier srcAttemptToMatch = srcAttempts.get((int)p);
    String filenameToMatch = SHUFFLE_INPUT_FILE_PREFIX + srcAttemptToMatch.getPathComponent();
    ArgumentCaptor<MapOutput> captureMapOutput = ArgumentCaptor.forClass(MapOutput.class);
    verify(scheduler).copySucceeded(eq(srcAttemptToMatch), eq(host), eq(p * 100),
        eq(p * 1000), anyLong(), captureMapOutput.capture());

    // cannot use the equals of MapOutput as it compares id which is private. so doing it manually
View Full Code Here
83
84
85
86
87
88
89
90
    byte[] bytes = new byte[(int)writer.getRawLength()];

    readDataToMem(inStream, bytes);
    inStream.close();

    InMemoryReader inMemReader = new InMemoryReader(null, new InputAttemptIdentifier(0, 0), bytes, 0, bytes.length);
    readAndVerify(inMemReader, data);
  }
View Full Code Here
115
116
117
118
119
120
121
122
    readDataToMem(inStream, bytes);
    inStream.close();


    InMemoryReader inMemReader = new InMemoryReader(null, new InputAttemptIdentifier(0, 0), bytes, 0, bytes.length);
    readAndVerify(inMemReader, data);
  }
View Full Code Here
146
147
148
149
150
151
152
153
154
155
156
//    MRTask mrTask = (MRTask)t.getProcessor();
//    Assert.assertEquals(TezNullOutputCommitter.class.getName(), mrTask
//        .getCommitter().getClass().getName());
//    t.close();

    Path mapOutputFile = mapOutputs.getInputFile(new InputAttemptIdentifier(0, 0));
    LOG.info("mapOutputFile = " + mapOutputFile);
    IFile.Reader reader =
        new IFile.Reader(localFs, mapOutputFile, null, null, null, false, 0, -1);
    LongWritable key = new LongWritable();
    Text value = new Text();
View Full Code Here
353
354
355
356
357
358
359
360
361
362
363
  }
 
  private void putBackRemainingMapOutputs(MapHost host) {
    // Cycle through remaining MapOutputs
    boolean isFirst = true;
    InputAttemptIdentifier first = null;
    for (InputAttemptIdentifier left : remaining) {
      if (isFirst) {
        first = left;
        isFirst = false;
        continue;
View Full Code Here
372
373
374
375
376
377
378
379
380
381
382
  private static InputAttemptIdentifier[] EMPTY_ATTEMPT_ID_ARRAY = new InputAttemptIdentifier[0];
 
  private InputAttemptIdentifier[] copyMapOutput(MapHost host,
                                DataInputStream input) {
    MapOutput mapOutput = null;
    InputAttemptIdentifier srcAttemptId = null;
    long decompressedLength = -1;
    long compressedLength = -1;
   
    try {
      long startTime = System.currentTimeMillis();
View Full Code Here
TOP

Related Classes of org.apache.tez.runtime.library.common.InputAttemptIdentifier

  • org.apache.tez.mapreduce.processor.map.TestMapProcessor
  • org.apache.tez.runtime.library.broadcast.input.BroadcastShuffleInputEventHandler
  • org.apache.tez.runtime.library.broadcast.input.BroadcastShuffleManager
  • org.apache.tez.runtime.library.broadcast.input.TestBroadcastInputManager
  • org.apache.tez.runtime.library.common.shuffle.impl.Fetcher
  • org.apache.tez.runtime.library.common.shuffle.impl.MergeManager$InMemoryMerger
  • org.apache.tez.runtime.library.common.shuffle.impl.MergeManager$IntermediateMemoryToMemoryMerger
  • org.apache.tez.runtime.library.common.shuffle.impl.ShuffleInputEventHandler
  • org.apache.tez.runtime.library.common.shuffle.impl.ShuffleScheduler
  • org.apache.tez.runtime.library.common.shuffle.impl.TestFetcher
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.