Package org.apache.hadoop.io.SequenceFile.Sorter

Examples of org.apache.hadoop.io.SequenceFile.Sorter.RawKeyValueIterator

804
805
806
807
808
809
810
811
812
813
814
          //is called (we delete empty sequence files as soon as we see them
          //in the merge method)
          SequenceFile.Writer writer = sorter.cloneFileAttributes(
              inMemFileSys.makeQualified(inMemClosedFiles[0]),
              localFileSys.makeQualified(inMemClosedFiles[0]), null);
          RawKeyValueIterator rIter;
          try {
            rIter = sorter.merge(inMemClosedFiles, true,
              inMemClosedFiles.length, new Path(reduceTask.getTaskId()));
          } catch (Exception e) {
            //make sure that we delete the ondisk file that we created earlier
View Full Code Here
666
667
668
669
670
671
672
673
674
675
676
        //in the merge method)
        SequenceFile.Writer writer = sorter.cloneFileAttributes(
            inMemFileSys.makeQualified(inMemClosedFiles[0]),
            localFileSys.makeQualified(inMemClosedFiles[0]), null);
       
        RawKeyValueIterator rIter = null;
        try {
          rIter = sorter.merge(inMemClosedFiles, true, inMemClosedFiles.length,
                       new Path(reduceTask.getTaskId()));
        } catch (Exception e) {
          //make sure that we delete the ondisk file that we created earlier
View Full Code Here
84
85
86
87
88
89
90
91
92
93
94
95
96
   
    SequenceFile.Sorter sorter = new SequenceFile.Sorter(fs,
        job.getOutputKeyComparator(), job.getMapOutputKeyClass(),
        job.getMapOutputValueClass(), job);
    Path[] paths = new Path[] {file};
    RawKeyValueIterator rIter = sorter.merge(paths, tempDir, false);
    int count = 0;
    while (rIter.next()) {
      count++;
    }
    assertEquals(RECORDS, count);
    assertEquals(1.0f, rIter.getProgress().get());
  }
View Full Code Here
598
599
600
601
602
603
604
605
606
607
608
        if (inMemClosedFiles.length == 0) {
          LOG.info(reduceTask.getTaskId() + "Nothing to merge from " +
              inMemFileSys.getUri());
          return numCopied == numOutputs;
        }
        RawKeyValueIterator rIter = sorter.merge(inMemClosedFiles, true,
                                                 inMemClosedFiles.length);
        //name this output file same as the name of the first file that is
        //there in the current list of inmem files (this is guaranteed to be
        //absent on the disk currently. So we don't overwrite a prev.
        //created spill)
View Full Code Here
719
720
721
722
723
724
725
726
727
728
729
        //Note that the above Path[] could be of length 0 if all copies are
        //in flight. So we make sure that we have some 'closed' map
        //output files to merge to get the benefit of in-memory merge
        if (inMemClosedFiles.length >=
          (int)(MAX_INMEM_FILESYS_USE/MAX_INMEM_FILESIZE_FRACTION)) {
          RawKeyValueIterator rIter = sorter.merge(inMemClosedFiles, true,
              inMemClosedFiles.length);
          //name this output file same as the name of the first file that is
          //there in the current list of inmem files (this is guaranteed to be
          //absent on the disk currently. So we don't overwrite a prev.
          //created spill)
View Full Code Here
TOP

Related Classes of org.apache.hadoop.io.SequenceFile.Sorter.RawKeyValueIterator

  • org.apache.hadoop.io.TestSequenceFileMergeProgress
  • org.apache.hadoop.mapred.MapTask$MapOutputBuffer
  • org.apache.hadoop.mapred.ReduceTaskRunner
  • org.apache.hadoop.mapred.ReduceTaskRunner$InMemFSMergeThread
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.