Package org.apache.spark.network.buffer

Examples of org.apache.spark.network.buffer.FileSegmentManagedBuffer


   * This logic is from FileShuffleBlockManager.
   */
  // TODO: Support consolidated hash shuffle files
  private ManagedBuffer getHashBasedShuffleBlockData(ExecutorShuffleInfo executor, String blockId) {
    File shuffleFile = getFile(executor.localDirs, executor.subDirsPerLocalDir, blockId);
    return new FileSegmentManagedBuffer(shuffleFile, 0, shuffleFile.length());
  }
View Full Code Here


    try {
      in = new DataInputStream(new FileInputStream(indexFile));
      in.skipBytes(reduceId * 8);
      long offset = in.readLong();
      long nextOffset = in.readLong();
      return new FileSegmentManagedBuffer(
        getFile(executor.localDirs, executor.subDirsPerLocalDir,
          "shuffle_" + shuffleId + "_" + mapId + "_0.data"),
        offset,
        nextOffset - offset);
    } catch (IOException e) {
View Full Code Here

    RandomAccessFile fp = new RandomAccessFile(testFile, "rw");
    byte[] fileContent = new byte[1024];
    new Random().nextBytes(fileContent);
    fp.write(fileContent);
    fp.close();
    fileChunk = new FileSegmentManagedBuffer(testFile, 10, testFile.length() - 25);

    TransportConf conf = new TransportConf(new SystemPropertyConfigProvider());
    streamManager = new StreamManager() {
      @Override
      public ManagedBuffer getChunk(long streamId, int chunkIndex) {
        assertEquals(STREAM_ID, streamId);
        if (chunkIndex == BUFFER_CHUNK_INDEX) {
          return new NioManagedBuffer(buf);
        } else if (chunkIndex == FILE_CHUNK_INDEX) {
          return new FileSegmentManagedBuffer(testFile, 10, testFile.length() - 25);
        } else {
          throw new IllegalArgumentException("Invalid chunk index: " + chunkIndex);
        }
      }
    };
View Full Code Here

TOP

Related Classes of org.apache.spark.network.buffer.FileSegmentManagedBuffer

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.