Examples of DistributedFileSystem

org.apache.hadoop.dfs.DistributedFileSystem
Implementation of the abstract FileSystem for the DFS system. This object is the way end-user code interacts with a Hadoop DistributedFileSystem.
org.apache.hadoop.hdfs.DistributedFileSystem
Implementation of the abstract FileSystem for the DFS system. This object is the way end-user code interacts with a Hadoop DistributedFileSystem.

Examples of org.apache.hadoop.hdfs.DistributedFileSystem

    if (!(fs instanceof DistributedFileSystem)) {
      System.err.println("FileSystem is " + fs.getUri());
      return exitCode;
    }


    DistributedFileSystem dfs = (DistributedFileSystem) fs;
    dfs.refreshNodes();
    exitCode = 0;
   
    return exitCode;
  }

View Full Code Here

Examples of org.apache.hadoop.hdfs.DistributedFileSystem

    if (!(fs instanceof DistributedFileSystem)) {
      System.out.println("FileSystem is " + fs.getUri());
      return exitCode;
    }


    DistributedFileSystem dfs = (DistributedFileSystem) fs;
    dfs.finalizeUpgrade();
    exitCode = 0;
   
    return exitCode;
  }

View Full Code Here

Examples of org.apache.hadoop.hdfs.DistributedFileSystem

    } else {
      printUsage("-upgradeProgress");
      return -1;
    }


    DistributedFileSystem dfs = (DistributedFileSystem) fs;
    UpgradeStatusReport status = dfs.distributedUpgradeProgress(action);
    String statusText = (status == null ? 
        "There are no upgrades in progress." :
          status.getStatusText(action == UpgradeAction.DETAILED_STATUS));
    System.out.println(statusText);
    return 0;

View Full Code Here

Examples of org.apache.hadoop.hdfs.DistributedFileSystem

   * @exception IOException if an error accoured wile accessing
   *            the file or path.
   */
  public int metaSave(String[] argv, int idx) throws IOException {
    String pathname = argv[idx];
    DistributedFileSystem dfs = (DistributedFileSystem) fs;
    dfs.metaSave(pathname);
    System.out.println("Created file " + pathname + " on server " +
                       dfs.getUri());
    return 0;
  }

View Full Code Here

Examples of org.apache.hadoop.hdfs.DistributedFileSystem

        
        DataOutputStream out = null;
        
        try {
          Configuration conf = new Configuration();
          DistributedFileSystem dfs = (DistributedFileSystem) FileSystem.get(conf);
          out = new DataOutputStream(new FileOutputStream(args[0]));
          UserGroupInformation ugi = UserGroupInformation.getCurrentUser();


          new DelegationTokenFetcher(dfs, out, ugi, conf).go();

View Full Code Here

Examples of org.apache.hadoop.hdfs.DistributedFileSystem

  /**
   * Tests save namepsace.
   */
  public void testSaveNamespace() throws IOException {
    MiniDFSCluster cluster = null;
    DistributedFileSystem fs = null;
    try {
      Configuration conf = new Configuration();
      cluster = new MiniDFSCluster(conf, numDatanodes, false, null);
      cluster.waitActive();
      fs = (DistributedFileSystem)(cluster.getFileSystem());


      // Saving image without safe mode should fail
      DFSAdmin admin = new DFSAdmin(conf);
      String[] args = new String[]{"-saveNamespace"};
      try {
        admin.run(args);
      } catch(IOException eIO) {
        assertTrue(eIO.getLocalizedMessage().contains("Safe mode should be turned ON"));
      } catch(Exception e) {
        throw new IOException(e);
      }
      // create new file
      Path file = new Path("namespace.dat");
      writeFile(fs, file, replication);
      checkFile(fs, file, replication);
      // verify that the edits file is NOT empty
      Collection<File> editsDirs = cluster.getNameEditsDirs();
      for(File ed : editsDirs) {
        assertTrue(new File(ed, "current/edits").length() > Integer.SIZE/Byte.SIZE);
      }


      // Saving image in safe mode should succeed
      fs.setSafeMode(SafeModeAction.SAFEMODE_ENTER);
      try {
        admin.run(args);
      } catch(Exception e) {
        throw new IOException(e);
      }
      // verify that the edits file is empty
      for(File ed : editsDirs) {
        assertTrue(new File(ed, "current/edits").length() == Integer.SIZE/Byte.SIZE);
      }


      // restart cluster and verify file exists
      cluster.shutdown();
      cluster = null;


      cluster = new MiniDFSCluster(conf, numDatanodes, false, null);
      cluster.waitActive();
      fs = (DistributedFileSystem)(cluster.getFileSystem());
      checkFile(fs, file, replication);
    } finally {
      if(fs != null) fs.close();
      if(cluster!= null) cluster.shutdown();
    }
  }

View Full Code Here

Examples of org.apache.hadoop.hdfs.DistributedFileSystem

  
  @Test
  public void testDelegationTokenDFSApi() throws Exception {
    DelegationTokenSecretManager dtSecretManager = cluster.getNameNode()
        .getNamesystem().getDelegationTokenSecretManager();
    DistributedFileSystem dfs = (DistributedFileSystem) cluster.getFileSystem();
    Token<DelegationTokenIdentifier> token = dfs.getDelegationToken(new Text("JobTracker"));
    DelegationTokenIdentifier identifier = new DelegationTokenIdentifier();
    byte[] tokenId = token.getIdentifier();
    identifier.readFields(new DataInputStream(
             new ByteArrayInputStream(tokenId)));
    Log.info("A valid token should have non-null password, and should be renewed successfully");

View Full Code Here

Examples of org.apache.hadoop.hdfs.DistributedFileSystem

    dtSecretManager.renewToken(token, "JobTracker");
  }
  
  @Test
  public void testDelegationTokenWithDoAs() throws Exception {
    final DistributedFileSystem dfs = (DistributedFileSystem) cluster.getFileSystem();
    final Token<DelegationTokenIdentifier> token = dfs.getDelegationToken(new Text(
        "JobTracker"));
    final UserGroupInformation longUgi = UserGroupInformation
        .createRemoteUser("JobTracker/foo.com@FOO.COM");
    final UserGroupInformation shortUgi = UserGroupInformation
        .createRemoteUser("JobTracker");
    longUgi.doAs(new PrivilegedExceptionAction<Object>() {
      public Object run() throws IOException {
        final DistributedFileSystem dfs = (DistributedFileSystem) cluster
            .getFileSystem();
        try {
          //try renew with long name
          dfs.renewDelegationToken(token);
        } catch (IOException e) {
          Assert.fail("Could not renew delegation token for user "+longUgi);
        }
        return null;
      }
    });
    shortUgi.doAs(new PrivilegedExceptionAction<Object>() {
      public Object run() throws IOException {
        final DistributedFileSystem dfs = (DistributedFileSystem) cluster
            .getFileSystem();
        dfs.renewDelegationToken(token);
        return null;
      }
    });
    longUgi.doAs(new PrivilegedExceptionAction<Object>() {
      public Object run() throws IOException {
        final DistributedFileSystem dfs = (DistributedFileSystem) cluster
            .getFileSystem();
        try {
          //try cancel with long name
          dfs.cancelDelegationToken(token);
        } catch (IOException e) {
          Assert.fail("Could not cancel delegation token for user "+longUgi);
        }
        return null;
      }

View Full Code Here

Examples of org.apache.hadoop.hdfs.DistributedFileSystem

   *  
   * @throws IOException
   */
  public void testManualSafeMode() throws IOException {
    MiniDFSCluster cluster = null;
    DistributedFileSystem fs = null;
    try {
      Configuration conf = new Configuration();
      // disable safemode extension to make the test run faster.
      conf.set("dfs.safemode.extension", "1");
      cluster = new MiniDFSCluster(conf, 1, true, null);
      cluster.waitActive();
      
      fs = (DistributedFileSystem)cluster.getFileSystem();
      Path file1 = new Path("/tmp/testManualSafeMode/file1");
      Path file2 = new Path("/tmp/testManualSafeMode/file2");
      
      LOG.info("Created file1 and file2.");
      
      // create two files with one block each.
      DFSTestUtil.createFile(fs, file1, 1000, (short)1, 0);
      DFSTestUtil.createFile(fs, file2, 2000, (short)1, 0);
      fs.close();
      cluster.shutdown();
      
      // now bring up just the NameNode.
      cluster = new MiniDFSCluster(conf, 0, false, null);
      cluster.waitActive();
      fs = (DistributedFileSystem)cluster.getFileSystem();
      
      LOG.info("Restarted cluster with just the NameNode");
      
      assertTrue("No datanode is started. Should be in SafeMode", 
                 fs.setSafeMode(SafeModeAction.SAFEMODE_GET));
      
      // manually set safemode.
      fs.setSafeMode(SafeModeAction.SAFEMODE_ENTER);
      
      // now bring up the datanode and wait for it to be active.
      cluster.startDataNodes(conf, 1, true, null, null);
      cluster.waitActive();
      
      LOG.info("Datanode is started.");


      // wait longer than dfs.safemode.extension
      try {
        Thread.sleep(2000);
      } catch (InterruptedException ignored) {}
      
      assertTrue("should still be in SafeMode",
          fs.setSafeMode(SafeModeAction.SAFEMODE_GET));
      
      fs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE);
      assertFalse("should not be in SafeMode",
          fs.setSafeMode(SafeModeAction.SAFEMODE_GET));
    } finally {
      if(fs != null) fs.close();
      if(cluster!= null) cluster.shutdown();
    }
  }

View Full Code Here

Examples of org.apache.hadoop.hdfs.DistributedFileSystem

   * Verify that the NameNode stays in safemode when dfs.safemode.datanode.min
   * is set to a number greater than the number of live datanodes.
   */
  public void testDatanodeThreshold() throws IOException {
    MiniDFSCluster cluster = null;
    DistributedFileSystem fs = null;
    try {
      Configuration conf = new Configuration();
      conf.set("dfs.safemode.extension", "0");
      conf.set("dfs.safemode.min.datanodes", "1");


      // bring up a cluster with no datanodes
      cluster = new MiniDFSCluster(conf, 0, false, null);
      cluster.waitActive();
      fs = (DistributedFileSystem)cluster.getFileSystem();


      assertTrue("No datanode started, but we require one - safemode expected",
                 fs.setSafeMode(SafeModeAction.SAFEMODE_GET));


      String tipMsg = cluster.getNameNode().getNamesystem().getSafeModeTip();
      assertTrue("Safemode tip message looks right",
                 tipMsg.contains("The number of live datanodes 0 needs an " +
                                 "additional 1 live"));


      // Start a datanode
      cluster.startDataNodes(conf, 1, true, null, null);


      // Wait long enough for safemode check to refire
      try {
        Thread.sleep(1000);
      } catch (InterruptedException ignored) {}


      // We now should be out of safe mode.
      assertFalse(
        "Out of safe mode after starting datanode.",
        fs.setSafeMode(SafeModeAction.SAFEMODE_GET));
    } finally {
      if (fs != null) fs.close();
      if (cluster != null) cluster.shutdown();
    }
  }

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.