Examples of DistributedFileSystem


Examples of org.apache.hadoop.dfs.DistributedFileSystem

    final int nameNodePort = cluster.getNameNodePort();

    // Get a reference to its DFS directly
    FileSystem fs = cluster.getFileSystem();
    assertTrue(fs instanceof DistributedFileSystem);
    DistributedFileSystem directDfs = (DistributedFileSystem) fs;

    // Get another reference via network using a specific socket factory
    Configuration cconf = new Configuration();
    cconf.set("fs.default.name", String.format("hdfs://localhost:%s/",
        nameNodePort + 10));
    cconf.set("hadoop.rpc.socket.factory.class.default",
        "org.apache.hadoop.ipc.DummySocketFactory");
    cconf.set("hadoop.rpc.socket.factory.class.ClientProtocol",
        "org.apache.hadoop.ipc.DummySocketFactory");
    cconf.set("hadoop.rpc.socket.factory.class.JobSubmissionProtocol",
        "org.apache.hadoop.ipc.DummySocketFactory");

    fs = FileSystem.get(cconf);
    assertTrue(fs instanceof DistributedFileSystem);
    DistributedFileSystem dfs = (DistributedFileSystem) fs;

    JobClient client = null;

    try {
      // This will test RPC to the NameNode only.
      // could we test Client-DataNode connections?
      Path filePath = new Path("/dir");

      assertFalse(directDfs.exists(filePath));
      assertFalse(dfs.exists(filePath));

      directDfs.mkdirs(filePath);
      assertTrue(directDfs.exists(filePath));
      assertTrue(dfs.exists(filePath));

      // This will test TPC to a JobTracker
      MiniMRCluster mr = new MiniMRCluster(1, fs.getUri().toString(), 1);
      final int jobTrackerPort = mr.getJobTrackerPort();

      JobConf jconf = new JobConf(cconf);
      jconf.set("mapred.job.tracker", String.format("localhost:%d",
          jobTrackerPort + 10));
      client = new JobClient(jconf);

      JobStatus[] jobs = client.jobsToComplete();
      assertTrue(jobs.length == 0);

    } finally {
      try {
        if (client != null)
          client.close();
      } catch (Exception ignored) {
        // nothing we can do
        ignored.printStackTrace();
      }
      try {
        if (dfs != null)
          dfs.close();

      } catch (Exception ignored) {
        // nothing we can do
        ignored.printStackTrace();
      }
View Full Code Here

Examples of org.apache.hadoop.hdfs.DistributedFileSystem

  throws IOException {
    if (!(fs instanceof DistributedFileSystem)) {
      return;
    }
    IOException exception = null;
    DistributedFileSystem dfs = (DistributedFileSystem) fs;
    try {
      if (dfs.exists(new Path("/"))) {
        return;
      }
    } catch (IOException e) {
      exception = RemoteExceptionHandler.checkIOException(e);
    }
View Full Code Here

Examples of org.apache.hadoop.hdfs.DistributedFileSystem

  public static void checkDfsSafeMode(final Configuration conf)
  throws IOException {
    boolean isInSafeMode = false;
    FileSystem fs = FileSystem.get(conf);
    if (fs instanceof DistributedFileSystem) {
      DistributedFileSystem dfs = (DistributedFileSystem)fs;
      isInSafeMode = isInSafeMode(dfs);
    }
    if (isInSafeMode) {
      throw new IOException("File system is in safemode, it can't be written now");
    }
View Full Code Here

Examples of org.apache.hadoop.hdfs.DistributedFileSystem

  public static void waitOnSafeMode(final Configuration conf,
    final long wait)
  throws IOException {
    FileSystem fs = FileSystem.get(conf);
    if (!(fs instanceof DistributedFileSystem)) return;
    DistributedFileSystem dfs = (DistributedFileSystem)fs;
    // Make sure dfs is not in safe mode
    while (isInSafeMode(dfs)) {
      LOG.info("Waiting for dfs to exit safe mode...");
      try {
        Thread.sleep(wait);
View Full Code Here

Examples of org.apache.hadoop.hdfs.DistributedFileSystem

    Token<DelegationTokenIdentifier> token = dttr.token;
    Configuration conf = dttr.conf;
   
    if(token.getKind().equals(kindHdfs)) {
      try {
        DistributedFileSystem dfs = getDFSForToken(token, conf);
        newExpirationDate = dfs.renewDelegationToken(token);
      } catch (InvalidToken ite) {
        LOG.warn("invalid token - not scheduling for renew");
        removeFailedDelegationToken(dttr);
        throw new IOException("failed to renew token", ite);
      } catch (AccessControlException ioe) {
View Full Code Here

Examples of org.apache.hadoop.hdfs.DistributedFileSystem

  }
 
  private static DistributedFileSystem getDFSForToken(
      Token<DelegationTokenIdentifier> token, final Configuration conf)
  throws Exception {
    DistributedFileSystem dfs = null;
    try {
      //TODO: The service is usually an IPaddress:port. We convert
      //it to dns name and then obtain the filesystem just so that
      //we reuse the existing filesystem handle (that the jobtracker
      //might have for this namenode; the namenode is usually
View Full Code Here

Examples of org.apache.hadoop.hdfs.DistributedFileSystem

    Token<DelegationTokenIdentifier> token = t.token;
    Configuration conf = t.conf;
   
    if(token.getKind().equals(kindHdfs)) {
      try {
        DistributedFileSystem dfs = getDFSForToken(token, conf);
        if (LOG.isDebugEnabled())
          LOG.debug("canceling token " + token.getService() + " for dfs=" +
              dfs);
        dfs.cancelDelegationToken(token);
      } catch (Exception e) {
        LOG.warn("Failed to cancel " + token, e);
      }
    }
  }
View Full Code Here

Examples of org.apache.hadoop.hdfs.DistributedFileSystem

   * Gives a report on how the FileSystem is doing.
   * @exception IOException if the filesystem does not exist.
   */
  public void report() throws IOException {
    if (fs instanceof DistributedFileSystem) {
      DistributedFileSystem dfs = (DistributedFileSystem) fs;
      FsStatus ds = dfs.getStatus();
      long capacity = ds.getCapacity();
      long used = ds.getUsed();
      long remaining = ds.getRemaining();
      long presentCapacity = used + remaining;
      boolean mode = dfs.setSafeMode(FSConstants.SafeModeAction.SAFEMODE_GET);
      UpgradeStatusReport status =
                      dfs.distributedUpgradeProgress(UpgradeAction.GET_STATUS);

      if (mode) {
        System.out.println("Safe mode is ON");
      }
      if (status != null) {
        System.out.println(status.getStatusText(false));
      }
      System.out.println("Configured Capacity: " + capacity
                         + " (" + StringUtils.byteDesc(capacity) + ")");
      System.out.println("Present Capacity: " + presentCapacity
          + " (" + StringUtils.byteDesc(presentCapacity) + ")");
      System.out.println("DFS Remaining: " + remaining
          + " (" + StringUtils.byteDesc(remaining) + ")");
      System.out.println("DFS Used: " + used
                         + " (" + StringUtils.byteDesc(used) + ")");
      System.out.println("DFS Used%: "
                         + StringUtils.limitDecimalTo2(((1.0 * used) / presentCapacity) * 100)
                         + "%");
     
      /* These counts are not always upto date. They are updated after 
       * iteration of an internal list. Should be updated in a few seconds to
       * minutes. Use "-metaSave" to list of all such blocks and accurate
       * counts.
       */
      System.out.println("Under replicated blocks: " +
                         dfs.getUnderReplicatedBlocksCount());
      System.out.println("Blocks with corrupt replicas: " +
                         dfs.getCorruptBlocksCount());
      System.out.println("Missing blocks: " +
                         dfs.getMissingBlocksCount());
                          
      System.out.println();

      System.out.println("-------------------------------------------------");
     
      DatanodeInfo[] live = dfs.getClient().datanodeReport(
                                                   DatanodeReportType.LIVE);
      DatanodeInfo[] dead = dfs.getClient().datanodeReport(
                                                   DatanodeReportType.DEAD);
      System.out.println("Datanodes available: " + live.length +
                         " (" + (live.length + dead.length) + " total, " +
                         dead.length + " dead)\n");
     
View Full Code Here

Examples of org.apache.hadoop.hdfs.DistributedFileSystem

      waitExitSafe = true;
    } else {
      printUsage("-safemode");
      return;
    }
    DistributedFileSystem dfs = (DistributedFileSystem) fs;
    boolean inSafeMode = dfs.setSafeMode(action);

    //
    // If we are waiting for safemode to exit, then poll and
    // sleep till we are out of safemode.
    //
    if (waitExitSafe) {
      while (inSafeMode) {
        try {
          Thread.sleep(5000);
        } catch (java.lang.InterruptedException e) {
          throw new IOException("Wait Interrupted");
        }
        inSafeMode = dfs.setSafeMode(action);
      }
    }

    System.out.println("Safe mode is " + (inSafeMode ? "ON" : "OFF"));
  }
View Full Code Here

Examples of org.apache.hadoop.hdfs.DistributedFileSystem

    if (!(fs instanceof DistributedFileSystem)) {
      System.err.println("FileSystem is " + fs.getUri());
      return exitCode;
    }

    DistributedFileSystem dfs = (DistributedFileSystem) fs;
    dfs.saveNamespace();
    exitCode = 0;
  
    return exitCode;
  }
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.