Package org.apache.hadoop.fs

Examples of org.apache.hadoop.fs.FSDataOutputStream.writeLong()

783
784
785
786
787
788
789
790
791
792
793
            // we need to close the writer to flush buffered data, obtaining
            // the correct offset
            writer.close();
            writer = null;
            indexOut.writeLong(segmentStart);
            indexOut.writeLong(out.getPos() - segmentStart);
          } finally {
            if (null != writer) writer.close();
          }
        }
        ++numSpills;
View Full Code Here
831
832
833
834
835
836
837
838
839
840
841
              // Note that our map byte count will not be accurate with
              // compression
              mapOutputByteCounter.increment(out.getPos() - recordStart);
            }
            writer.close();
            indexOut.writeLong(segmentStart);
            indexOut.writeLong(out.getPos() - segmentStart);
          } catch (IOException e) {
            if (null != writer) writer.close();
            throw e;
          }
View Full Code Here
832
833
834
835
836
837
838
839
840
841
842
              // compression
              mapOutputByteCounter.increment(out.getPos() - recordStart);
            }
            writer.close();
            indexOut.writeLong(segmentStart);
            indexOut.writeLong(out.getPos() - segmentStart);
          } catch (IOException e) {
            if (null != writer) writer.close();
            throw e;
          }
        }
View Full Code Here
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
          segmentStart = finalOut.getPos();
          Writer writer = SequenceFile.createWriter(job, finalOut,
                                                    job.getMapOutputKeyClass(),
                                                    job.getMapOutputValueClass(),
                                                    compressionType, codec);
          finalIndexOut.writeLong(segmentStart);
          finalIndexOut.writeLong(finalOut.getPos() - segmentStart);
          writer.close();
        }
        finalOut.close();
        finalIndexOut.close();
View Full Code Here
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
          Writer writer = SequenceFile.createWriter(job, finalOut,
                                                    job.getMapOutputKeyClass(),
                                                    job.getMapOutputValueClass(),
                                                    compressionType, codec);
          finalIndexOut.writeLong(segmentStart);
          finalIndexOut.writeLong(finalOut.getPos() - segmentStart);
          writer.close();
        }
        finalOut.close();
        finalIndexOut.close();
        return;
View Full Code Here
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
          //when we write the offset/length to the final index file, we write
          //longs for both. This helps us to reliably seek directly to the
          //offset/length for a partition when we start serving the byte-ranges
          //to the reduces. We probably waste some space in the file by doing
          //this as opposed to writing VLong but it helps us later on.
          finalIndexOut.writeLong(segmentStart);
          finalIndexOut.writeLong(finalOut.getPos()-segmentStart);
        }
        finalOut.close();
        finalIndexOut.close();
        //cleanup
View Full Code Here
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
          //longs for both. This helps us to reliably seek directly to the
          //offset/length for a partition when we start serving the byte-ranges
          //to the reduces. We probably waste some space in the file by doing
          //this as opposed to writing VLong but it helps us later on.
          finalIndexOut.writeLong(segmentStart);
          finalIndexOut.writeLong(finalOut.getPos()-segmentStart);
        }
        finalOut.close();
        finalIndexOut.close();
        //cleanup
        for(int i = 0; i < numSpills; i++) {
View Full Code Here
234
235
236
237
238
239
240
241
242
243
244
          LinkedList<Long> offsetList = rootMap.get(key);
          if (offsetList.size() > 1 || offsetList.size() == 0) {
            throw new IOException("Why root index doen't have one offset?");
          }
          rootOut.writeLong(offsetList.getFirst());

        }
        rootOut.flush();
        rootOut.close();
View Full Code Here
342
343
344
345
346
347
348
349
350
351
352
  void writeInfo(FileSystem fs, long infonum) throws IOException {
    Path p = getInfoFilePath();
    FSDataOutputStream out = fs.create(p);
    try {
      out.writeByte(INFO_SEQ_NUM);
      out.writeLong(infonum);
    } finally {
      out.close();
    }
  }
 
View Full Code Here
645
646
647
648
649
650
651
652
653
654
655
    CheckedOutputStream chk = new CheckedOutputStream(output, crc);
    String msg = "Writing new index file. This file will be used only " +
        "for the testing.";
    chk.write(Arrays.copyOf(msg.getBytes(),
        MapTask.MAP_OUTPUT_INDEX_RECORD_LENGTH));
    output.writeLong(chk.getChecksum().getValue());
    output.close();
  }

  @Test
  public void testRecovery() throws IOException {
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.