Examples of org.apache.hadoop.io.nativeio.SharedFileDescriptorFactory

org.apache.hadoop.io.nativeio.SharedFileDescriptorFactory
A factory for creating shared file descriptors inside a given directory. Typically, the directory will be /dev/shm or /tmp. We will hand out file descriptors that correspond to unlinked files residing in that directory. These file descriptors are suitable for sharing across multiple processes and are both readable and writable. Because we unlink the temporary files right after creating them, a JVM crash usually does not leave behind any temporary files in the directory. However, it may happen that we crash right after creating the file and before unlinking it. In the constructor, we attempt to clean up after any such remnants by trying to unlink any temporary files created by previous SharedFileDescriptorFactory instances that also used our prefix.


  @Test(timeout=60000)
  public void testStartupShutdown() throws Exception {
    File path = new File(TEST_BASE, "testStartupShutdown");
    path.mkdirs();
    SharedFileDescriptorFactory factory =
        SharedFileDescriptorFactory.create("shm_",
            new String[] { path.getAbsolutePath() } );
    FileInputStream stream =
        factory.createDescriptor("testStartupShutdown", 4096);
    ShortCircuitShm shm = new ShortCircuitShm(ShmId.createRandom(), stream);
    shm.free();
    stream.close();
    FileUtil.fullyDelete(path);
  }

View Full Code Here


  @Test(timeout=60000)
  public void testAllocateSlots() throws Exception {
    File path = new File(TEST_BASE, "testAllocateSlots");
    path.mkdirs();
    SharedFileDescriptorFactory factory =
        SharedFileDescriptorFactory.create("shm_", 
            new String[] { path.getAbsolutePath() });
    FileInputStream stream =
        factory.createDescriptor("testAllocateSlots", 4096);
    ShortCircuitShm shm = new ShortCircuitShm(ShmId.createRandom(), stream);
    int numSlots = 0;
    ArrayList<Slot> slots = new ArrayList<Slot>();
    while (!shm.isFull()) {
      Slot slot = shm.allocAndRegisterSlot(new ExtendedBlockId(123L, "test_bp1"));

View Full Code Here

  private final HashMultimap<ExtendedBlockId, Slot> slots =
      HashMultimap.create(0, 1);
  
  public ShortCircuitRegistry(Configuration conf) throws IOException {
    boolean enabled = false;
    SharedFileDescriptorFactory shmFactory = null;
    DomainSocketWatcher watcher = null;
    try {
      int interruptCheck = conf.getInt(
          DFS_SHORT_CIRCUIT_SHARED_MEMORY_WATCHER_INTERRUPT_CHECK_MS,
          DFS_SHORT_CIRCUIT_SHARED_MEMORY_WATCHER_INTERRUPT_CHECK_MS_DEFAULT);
      if (interruptCheck <= 0) {
        throw new IOException(
            DFS_SHORT_CIRCUIT_SHARED_MEMORY_WATCHER_INTERRUPT_CHECK_MS +
            " was set to " + interruptCheck);
      }
      String shmPaths[] =
          conf.getTrimmedStrings(DFS_DATANODE_SHARED_FILE_DESCRIPTOR_PATHS);
      if (shmPaths.length == 0) {
        shmPaths =
            DFS_DATANODE_SHARED_FILE_DESCRIPTOR_PATHS_DEFAULT.split(",");
      }
      shmFactory = SharedFileDescriptorFactory.
          create("HadoopShortCircuitShm_", shmPaths);
      String dswLoadingFailure = DomainSocketWatcher.getLoadingFailureReason();
      if (dswLoadingFailure != null) {
        throw new IOException(dswLoadingFailure);
      }
      watcher = new DomainSocketWatcher(interruptCheck);
      enabled = true;
      if (LOG.isDebugEnabled()) {
        LOG.debug("created new ShortCircuitRegistry with interruptCheck=" +
                  interruptCheck + ", shmPath=" + shmFactory.getPath());
      }
    } catch (IOException e) {
      if (LOG.isDebugEnabled()) {
        LOG.debug("Disabling ShortCircuitRegistry", e);
      }

View Full Code Here


  @Test(timeout=60000)
  public void testStartupShutdown() throws Exception {
    File path = new File(TEST_BASE, "testStartupShutdown");
    path.mkdirs();
    SharedFileDescriptorFactory factory =
        SharedFileDescriptorFactory.create("shm_",
            new String[] { path.getAbsolutePath() } );
    FileInputStream stream =
        factory.createDescriptor("testStartupShutdown", 4096);
    ShortCircuitShm shm = new ShortCircuitShm(ShmId.createRandom(), stream);
    shm.free();
    stream.close();
    FileUtil.fullyDelete(path);
  }

View Full Code Here


  @Test(timeout=60000)
  public void testAllocateSlots() throws Exception {
    File path = new File(TEST_BASE, "testAllocateSlots");
    path.mkdirs();
    SharedFileDescriptorFactory factory =
        SharedFileDescriptorFactory.create("shm_", 
            new String[] { path.getAbsolutePath() });
    FileInputStream stream =
        factory.createDescriptor("testAllocateSlots", 4096);
    ShortCircuitShm shm = new ShortCircuitShm(ShmId.createRandom(), stream);
    int numSlots = 0;
    ArrayList<Slot> slots = new ArrayList<Slot>();
    while (!shm.isFull()) {
      Slot slot = shm.allocAndRegisterSlot(new ExtendedBlockId(123L, "test_bp1"));

View Full Code Here

  private final HashMultimap<ExtendedBlockId, Slot> slots =
      HashMultimap.create(0, 1);
  
  public ShortCircuitRegistry(Configuration conf) throws IOException {
    boolean enabled = false;
    SharedFileDescriptorFactory shmFactory = null;
    DomainSocketWatcher watcher = null;
    try {
      int interruptCheck = conf.getInt(
          DFS_SHORT_CIRCUIT_SHARED_MEMORY_WATCHER_INTERRUPT_CHECK_MS,
          DFS_SHORT_CIRCUIT_SHARED_MEMORY_WATCHER_INTERRUPT_CHECK_MS_DEFAULT);
      if (interruptCheck <= 0) {
        throw new IOException(
            DFS_SHORT_CIRCUIT_SHARED_MEMORY_WATCHER_INTERRUPT_CHECK_MS +
            " was set to " + interruptCheck);
      }
      String shmPaths[] =
          conf.getTrimmedStrings(DFS_DATANODE_SHARED_FILE_DESCRIPTOR_PATHS);
      if (shmPaths.length == 0) {
        shmPaths =
            DFS_DATANODE_SHARED_FILE_DESCRIPTOR_PATHS_DEFAULT.split(",");
      }
      shmFactory = SharedFileDescriptorFactory.
          create("HadoopShortCircuitShm_", shmPaths);
      String dswLoadingFailure = DomainSocketWatcher.getLoadingFailureReason();
      if (dswLoadingFailure != null) {
        throw new IOException(dswLoadingFailure);
      }
      watcher = new DomainSocketWatcher(interruptCheck);
      enabled = true;
      if (LOG.isDebugEnabled()) {
        LOG.debug("created new ShortCircuitRegistry with interruptCheck=" +
                  interruptCheck + ", shmPath=" + shmFactory.getPath());
      }
    } catch (IOException e) {
      if (LOG.isDebugEnabled()) {
        LOG.debug("Disabling ShortCircuitRegistry", e);
      }

View Full Code Here

TOP

Related Classes of org.apache.hadoop.io.nativeio.SharedFileDescriptorFactory

org.apache.hadoop.hdfs.client.TestShortCircuitShm

org.apache.hadoop.hdfs.server.datanode.ShortCircuitRegistry

org.apache.hadoop.hdfs.shortcircuit.TestShortCircuitShm

java.io.IOException

java.io.FileInputStream

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.