Package org.apache.hadoop.io.SequenceFile.Sorter

Examples of org.apache.hadoop.io.SequenceFile.Sorter.RawKeyValueIterator


        mapOutputFile.getSpillFile(getTaskId(), numSpills).getName());
         
        //invoke the sort
        for (int i = 0; i < partitions; i++) {
          sortImpl[i].setInputBuffer(keyValBuffer);
          RawKeyValueIterator rIter = sortImpl[i].sort();
         
          startPartition(i);
          if (rIter != null) {
            //invoke the combiner if one is defined
            if (job.getCombinerClass() != null) {
View Full Code Here


        //in the merge method)
        SequenceFile.Writer writer = sorter.cloneFileAttributes(
            inMemFileSys.makeQualified(inMemClosedFiles[0]),
            localFileSys.makeQualified(inMemClosedFiles[0]), null);
       
        RawKeyValueIterator rIter = null;
        try {
          rIter = sorter.merge(inMemClosedFiles, true, inMemClosedFiles.length,
                       new Path(reduceTask.getTaskId()));
        } catch (Exception e) {
          //make sure that we delete the ondisk file that we created earlier
View Full Code Here

          //is called (we delete empty sequence files as soon as we see them
          //in the merge method)
          SequenceFile.Writer writer = sorter.cloneFileAttributes(
              inMemFileSys.makeQualified(inMemClosedFiles[0]),
              localFileSys.makeQualified(inMemClosedFiles[0]), null);
          RawKeyValueIterator rIter;
          try {
            rIter = sorter.merge(inMemClosedFiles, true,
              inMemClosedFiles.length, new Path(reduceTask.getTaskId()));
          } catch (Exception e) {
            //make sure that we delete the ondisk file that we created earlier
View Full Code Here

         
        //invoke the sort
        for (int i = 0; i < partitions; i++) {
          sortImpl[i].setInputBuffer(keyValBuffer);
          sortImpl[i].setProgressable(reporter);
          RawKeyValueIterator rIter = sortImpl[i].sort();
         
          startPartition(i);
          if (rIter != null) {
            //invoke the combiner if one is defined
            if (job.getCombinerClass() != null) {
View Full Code Here

            s.preserveInput(true);
            s.doSync();
            segmentList.add(i, s);
          }
          segmentStart = finalOut.getPos();
          RawKeyValueIterator kvIter = sorter.merge(segmentList, new Path(getTaskId()));
          SequenceFile.Writer writer = SequenceFile.createWriter(job, finalOut,
                                                                 job.getMapOutputKeyClass(), job.getMapOutputValueClass(),
                                                                 compressionType, codec);
          sorter.writeFile(kvIter, writer);
          //close the file - required esp. for block compression to ensure
View Full Code Here

        if (inMemClosedFiles.length == 0) {
          LOG.info(reduceTask.getTaskId() + "Nothing to merge from " +
              inMemFileSys.getUri());
          return numCopied == numOutputs;
        }
        RawKeyValueIterator rIter =
          sorter.merge(inMemClosedFiles, true, inMemClosedFiles.length,
                       new Path(reduceTask.getTaskId()));
        //name this output file same as the name of the first file that is
        //there in the current list of inmem files (this is guaranteed to be
        //absent on the disk currently. So we don't overwrite a prev.
View Full Code Here

        //Note that the above Path[] could be of length 0 if all copies are
        //in flight. So we make sure that we have some 'closed' map
        //output files to merge to get the benefit of in-memory merge
        if (inMemClosedFiles.length >=
          (int)(MAX_INMEM_FILESYS_USE/MAX_INMEM_FILESIZE_FRACTION)) {
          RawKeyValueIterator rIter = sorter.merge(inMemClosedFiles, true,
              inMemClosedFiles.length, new Path(reduceTask.getTaskId()));
          //name this output file same as the name of the first file that is
          //there in the current list of inmem files (this is guaranteed to be
          //absent on the disk currently. So we don't overwrite a prev.
          //created spill)
View Full Code Here

         
        //invoke the sort
        for (int i = 0; i < partitions; i++) {
          sortImpl[i].setInputBuffer(keyValBuffer);
          sortImpl[i].setProgressable(reporter);
          RawKeyValueIterator rIter = sortImpl[i].sort();
         
          startPartition(i);
          if (rIter != null) {
            //invoke the combiner if one is defined
            if (job.getCombinerClass() != null) {
View Full Code Here

            s.preserveInput(true);
            s.doSync();
            segmentList.add(i, s);
          }
          segmentStart = finalOut.getPos();
          RawKeyValueIterator kvIter = sorter.merge(segmentList, new Path(getTaskId()));
          SequenceFile.Writer writer = SequenceFile.createWriter(job, finalOut,
                                                                 job.getMapOutputKeyClass(), job.getMapOutputValueClass(),
                                                                 compressionType, codec);
          sorter.writeFile(kvIter, writer);
          //close the file - required esp. for block compression to ensure
View Full Code Here

            s.preserveInput(true);
            s.doSync();
            segmentList.add(i, s);
          }
          segmentStart = finalOut.getPos();
          RawKeyValueIterator kvIter = sorter.merge(segmentList, new Path(getTaskId()));
          SequenceFile.Writer writer = SequenceFile.createWriter(job, finalOut,
                                                                 job.getMapOutputKeyClass(), job.getMapOutputValueClass(),
                                                                 compressionType, codec);
          sorter.writeFile(kvIter, writer);
          //close the file - required esp. for block compression to ensure
View Full Code Here

TOP

Related Classes of org.apache.hadoop.io.SequenceFile.Sorter.RawKeyValueIterator

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.