Examples of InputAttemptIdentifier

Examples of org.apache.tez.runtime.library.common.InputAttemptIdentifier

54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
   
    long requestSize = (long) (0.4f * inMemThreshold);
    long compressedSize = 1l;
    LOG.info("RequestSize: " + requestSize);
   
    FetchedInput fi1 = inputManager.allocate(requestSize, compressedSize, new InputAttemptIdentifier(1, 1));
    assertEquals(FetchedInput.Type.MEMORY, fi1.getType());
   
   
    FetchedInput fi2 = inputManager.allocate(requestSize, compressedSize, new InputAttemptIdentifier(2, 1));
    assertEquals(FetchedInput.Type.MEMORY, fi2.getType());
   
   
    // Over limit by this point. Next reserve should give back a DISK allocation
    FetchedInput fi3 = inputManager.allocate(requestSize, compressedSize, new InputAttemptIdentifier(3, 1));
    assertEquals(FetchedInput.Type.DISK, fi3.getType());
   
   
    // Freed one memory allocation. Next should be mem again.
    fi1.abort();
    fi1.free();
    FetchedInput fi4 = inputManager.allocate(requestSize, compressedSize, new InputAttemptIdentifier(4, 1));
    assertEquals(FetchedInput.Type.MEMORY, fi4.getType());
   
    // Freed one disk allocation. Next sould be disk again (no mem freed)
    fi3.abort();
    fi3.free();
    FetchedInput fi5 = inputManager.allocate(requestSize, compressedSize, new InputAttemptIdentifier(4, 1));
    assertEquals(FetchedInput.Type.DISK, fi5.getType());
  }
View Full Code Here

Examples of org.apache.tez.runtime.library.common.InputAttemptIdentifier

294
295
296
297
298
299
300
301
302
303
304
305
306
    // remove from the obsolete list.
    List<InputAttemptIdentifier> pendingInputsForHost = inputHost
        .clearAndGetPendingInputs();
    for (Iterator<InputAttemptIdentifier> inputIter = pendingInputsForHost
        .iterator(); inputIter.hasNext();) {
      InputAttemptIdentifier input = inputIter.next();
      // Avoid adding attempts which have already completed.
      if (completedInputSet.contains(input.getInputIdentifier())) {
        inputIter.remove();
      }
      // Avoid adding attempts which have been marked as OBSOLETE
      if (obsoletedInputs.contains(input)) {
        inputIter.remove();
View Full Code Here

Examples of org.apache.tez.runtime.library.common.InputAttemptIdentifier

300
301
302
303
304
305
306
307
308
309
310
        }
      } else {
        // If we got a read error at this stage, it implies there was a problem
        // with the first map, typically lost map. So, penalize only that map
        // and add the rest
        InputAttemptIdentifier firstMap = srcAttempts.get(0);
        scheduler.copyFailed(firstMap, host, connectSucceeded);
      }
     
      // Add back all the remaining maps, WITHOUT marking them as failed
      for(InputAttemptIdentifier left: remaining) {
View Full Code Here

Examples of org.apache.tez.runtime.library.common.InputAttemptIdentifier

349
350
351
352
353
354
355
356
357
358
359
  private static InputAttemptIdentifier[] EMPTY_ATTEMPT_ID_ARRAY = new InputAttemptIdentifier[0];
 
  private InputAttemptIdentifier[] copyMapOutput(MapHost host,
                                DataInputStream input) {
    MapOutput mapOutput = null;
    InputAttemptIdentifier srcAttemptId = null;
    long decompressedLength = -1;
    long compressedLength = -1;
   
    try {
      long startTime = System.currentTimeMillis();
View Full Code Here

Examples of org.apache.tez.runtime.library.common.InputAttemptIdentifier

72
73
74
75
76
77
78
79
80
81
82
    } catch (InvalidProtocolBufferException e) {
      throw new TezUncheckedException("Unable to parse DataMovementEvent payload", e);
    }
    int partitionId = dmEvent.getSourceIndex();
    URI baseUri = getBaseURI(shufflePayload.getHost(), shufflePayload.getPort(), partitionId);
    InputAttemptIdentifier srcAttemptIdentifier = new InputAttemptIdentifier(dmEvent.getTargetIndex(), dmEvent.getVersion(), shufflePayload.getPathComponent());
    LOG.info("DataMovementEvent baseUri:" + baseUri + ", src: " + srcAttemptIdentifier);
    scheduler.addKnownMapOutput(shufflePayload.getHost(), partitionId, baseUri.toString(), srcAttemptIdentifier);
   
    // TODO NEWTEZ See if this duration hack can be removed.
    int duration = shufflePayload.getRunDuration();
View Full Code Here

Examples of org.apache.tez.runtime.library.common.InputAttemptIdentifier

85
86
87
88
89
90
91
92
93
      scheduler.informMaxMapRunTime(maxMapRuntime);
    }
  }
 
  private void processTaskFailedEvent(InputFailedEvent ifEvent) {
    InputAttemptIdentifier taIdentifier = new InputAttemptIdentifier(ifEvent.getSourceIndex(), ifEvent.getVersion());
    scheduler.obsoleteMapOutput(taIdentifier);
    LOG.info("Obsoleting output of src-task: " + taIdentifier);
  }
View Full Code Here

Examples of org.apache.tez.runtime.library.common.InputAttemptIdentifier

399
400
401
402
403
404
405
406
407
408
409
    public void merge(List<MapOutput> inputs) throws IOException {
      if (inputs == null || inputs.size() == 0) {
        return;
      }

      InputAttemptIdentifier dummyMapId = inputs.get(0).getAttemptIdentifier();
      List<Segment> inMemorySegments = new ArrayList<Segment>();
      long mergeOutputSize =
        createInMemorySegments(inputs, inMemorySegments, 0);
      int noInMemorySegments = inMemorySegments.size();
     
View Full Code Here

Examples of org.apache.tez.runtime.library.common.InputAttemptIdentifier

458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
      //it is not guaranteed that this file will be present after merge
      //is called (we delete empty files as soon as we see them
      //in the merge method)

      //figure out the mapId
      InputAttemptIdentifier srcTaskIdentifier = inputs.get(0).getAttemptIdentifier();

      List<Segment> inMemorySegments = new ArrayList<Segment>();
      long mergeOutputSize =
        createInMemorySegments(inputs, inMemorySegments,0);
      int noInMemorySegments = inMemorySegments.size();

      Path outputPath = mapOutputFile.getInputFileForWrite(
          srcTaskIdentifier.getInputIdentifier().getSrcTaskIndex(),
          mergeOutputSize).suffix(Constants.MERGED_OUTPUT_PREFIX);

      Writer writer = null;
      try {
        writer =
View Full Code Here

Examples of org.apache.tez.runtime.library.common.InputAttemptIdentifier

90
91
92
93
94
95
96
97
98
99
100
    int srcIdx = 0;
    int targetIdx = 1;
    Event dme = createDataMovementEvent(srcIdx, targetIdx, null, false);
    events.add(dme);
    handler.handleEvents(events);
    InputAttemptIdentifier expectedIdentifier = new InputAttemptIdentifier(targetIdx, 0,
        PATH_COMPONENT);

    String baseUri = handler.getBaseURI(HOST, PORT, srcIdx).toString();
    int partitionId = srcIdx;
    verify(scheduler).addKnownMapOutput(eq(HOST), eq(PORT), eq(partitionId),
View Full Code Here

Examples of org.apache.tez.runtime.library.common.InputAttemptIdentifier

106
107
108
109
110
111
112
113
    List<Event> events = new LinkedList<Event>();
    int targetIdx = 1;
    InputFailedEvent failedEvent = InputFailedEvent.create(targetIdx, 0);
    events.add(failedEvent);
    handler.handleEvents(events);
    InputAttemptIdentifier expectedIdentifier = new InputAttemptIdentifier(targetIdx, 0);
    verify(scheduler).obsoleteInput(eq(expectedIdentifier));
  }
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.