Examples of StorageDirectory


Examples of org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory

    try {
      List<Thread> saveThreads = new ArrayList<Thread>();
      // save images into current
      for (Iterator<StorageDirectory> it
             = storage.dirIterator(NameNodeDirType.IMAGE); it.hasNext();) {
        StorageDirectory sd = it.next();
        FSImageSaver saver = new FSImageSaver(ctx, sd);
        Thread saveThread = new Thread(saver, saver.toString());
        saveThreads.add(saveThread);
        saveThread.start();
      }
View Full Code Here

Examples of org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory

      // FS Image storage configuration
      out.print("<h3> " + nn.getRole() + " Storage: </h3>");
      out.print("<div class=\"dfstable\"> <table class=\"storage\" title=\"NameNode Storage\">\n"
              + "<thead><tr><td><b>Storage Directory</b></td><td><b>Type</b></td><td><b>State</b></td></tr></thead>");

      StorageDirectory st = null;
      for (Iterator<StorageDirectory> it
             = fsImage.getStorage().dirIterator(); it.hasNext();) {
        st = it.next();
        String dir = "" + st.getRoot();
        String type = "" + st.getStorageDirType();
        out.print("<tr><td>" + dir + "</td><td>" + type
            + "</td><td>Active</td></tr>");
      }

      long storageDirsSize = removedStorageDirs.size();
      for (int i = 0; i < storageDirsSize; i++) {
        st = removedStorageDirs.get(i);
        String dir = "" + st.getRoot();
        String type = "" + st.getStorageDirType();
        out.print("<tr><td>" + dir + "</td><td>" + type
            + "</td><td><span class=\"failed\">Failed</span></td></tr>");
      }

      out.print("</table></div>\n");
View Full Code Here

Examples of org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory

      storage.attemptRestoreRemovedStorage();
      storage.unlockAll();

      for (Iterator<StorageDirectory> it =
                   storage.dirIterator(); it.hasNext();) {
        StorageDirectory sd = it.next();
        boolean isAccessible = true;
        try { // create directories if don't exist yet
          if(!sd.getRoot().mkdirs()) {
            // do nothing, directory is already created
          }
        } catch(SecurityException se) {
          isAccessible = false;
        }
        if(!isAccessible)
          throw new InconsistentFSStateException(sd.getRoot(),
              "cannot access checkpoint directory.");
       
        if (format) {
          // Don't confirm, since this is just the secondary namenode.
          LOG.info("Formatting storage directory " + sd);
          sd.clearDirectory();
        }
       
        StorageState curState;
        try {
          curState = sd.analyzeStorage(HdfsServerConstants.StartupOption.REGULAR, storage);
          // sd is locked but not opened
          switch(curState) {
          case NON_EXISTENT:
            // fail if any of the configured checkpoint dirs are inaccessible
            throw new InconsistentFSStateException(sd.getRoot(),
                  "checkpoint directory does not exist or is not accessible.");
          case NOT_FORMATTED:
            break// it's ok since initially there is no current and VERSION
          case NORMAL:
            // Read the VERSION file. This verifies that:
            // (a) the VERSION file for each of the directories is the same,
            // and (b) when we connect to a NN, we can verify that the remote
            // node matches the same namespace that we ran on previously.
            storage.readProperties(sd);
            break;
          default// recovery is possible
            sd.doRecover(curState);
          }
        } catch (IOException ioe) {
          sd.unlock();
          throw ioe;
        }
      }
    }
View Full Code Here

Examples of org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory

     * directories
     */
    void ensureCurrentDirExists() throws IOException {
      for (Iterator<StorageDirectory> it
             = storage.dirIterator(); it.hasNext();) {
        StorageDirectory sd = it.next();
        File curDir = sd.getCurrentDir();
        if (!curDir.exists() && !curDir.mkdirs()) {
          throw new IOException("Could not create directory " + curDir);
        }
      }
    }
View Full Code Here

Examples of org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory

    for (URI u : dirs) {
      boolean required = FSNamesystem.getRequiredNamespaceEditsDirs(conf)
          .contains(u);
      if (u.getScheme().equals(NNStorage.LOCAL_URI_SCHEME)) {
        StorageDirectory sd = storage.getStorageDirectory(u);
        if (sd != null) {
          journalSet.add(new FileJournalManager(sd, storage), required);
        }
      } else {
        journalSet.add(createJournal(u), required);
View Full Code Here

Examples of org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory

    boolean canRollback = false;
    FSImage prevState = new FSImage(conf);
    try {
      prevState.getStorage().layoutVersion = HdfsConstants.LAYOUT_VERSION;
      for (Iterator<StorageDirectory> it = storage.dirIterator(); it.hasNext();) {
        StorageDirectory sd = it.next();
        File prevDir = sd.getPreviousDir();
        if (!prevDir.exists()) {  // use current directory then
          LOG.info("Storage directory " + sd.getRoot()
            + " does not contain previous fs state.");
          // read and verify consistency with other directories
          storage.readProperties(sd);
          continue;
        }

        // read and verify consistency of the prev dir
        prevState.getStorage().readPreviousVersionProperties(sd);

        if (prevState.getLayoutVersion() != HdfsConstants.LAYOUT_VERSION) {
          throw new IOException(
            "Cannot rollback to storage version " +
                prevState.getLayoutVersion() +
                " using this version of the NameNode, which uses storage version " +
                HdfsConstants.LAYOUT_VERSION + ". " +
              "Please use the previous version of HDFS to perform the rollback.");
        }
        canRollback = true;
      }
      if (!canRollback)
        throw new IOException("Cannot rollback. None of the storage "
            + "directories contain previous fs state.");

      // Now that we know all directories are going to be consistent
      // Do rollback for each directory containing previous state
      for (Iterator<StorageDirectory> it = storage.dirIterator(); it.hasNext();) {
        StorageDirectory sd = it.next();
        File prevDir = sd.getPreviousDir();
        if (!prevDir.exists())
          continue;

        LOG.info("Rolling back storage directory " + sd.getRoot()
          + ".\n   new LV = " + prevState.getStorage().getLayoutVersion()
          + "; new CTime = " + prevState.getStorage().getCTime());
        File tmpDir = sd.getRemovedTmp();
        assert !tmpDir.exists() : "removed.tmp directory must not exist.";
        // rename current to tmp
        File curDir = sd.getCurrentDir();
        assert curDir.exists() : "Current directory must exist.";
        NNStorage.rename(curDir, tmpDir);
        // rename previous to current
        NNStorage.rename(prevDir, curDir);

        // delete tmp dir
        NNStorage.deleteDir(tmpDir);
        LOG.info("Rollback of " + sd.getRoot()+ " is complete.");
      }
      isUpgradeFinalized = true;
    } finally {
      prevState.close();
    }
View Full Code Here

Examples of org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory

    getStorage().writeAll();
  }

  void finalizeUpgrade() throws IOException {
    for (Iterator<StorageDirectory> it = storage.dirIterator(); it.hasNext();) {
      StorageDirectory sd = it.next();
      doFinalize(sd);
    }
  }
View Full Code Here

Examples of org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory

  }

  void loadFSImageFile(FSNamesystem target, MetaRecoveryContext recovery,
      FSImageFile imageFile) throws IOException {
    LOG.debug("Planning to load image :\n" + imageFile);
    StorageDirectory sdForProperties = imageFile.sd;
    storage.readProperties(sdForProperties);

    if (LayoutVersion.supports(Feature.TXID_BASED_LAYOUT,
                               getLayoutVersion())) {
      // For txid-based layout, we should have a .md5 file
      // next to the image file
      loadFSImage(imageFile.getFile(), target, recovery);
    } else if (LayoutVersion.supports(Feature.FSIMAGE_CHECKSUM,
                                      getLayoutVersion())) {
      // In 0.22, we have the checksum stored in the VERSION file.
      String md5 = storage.getDeprecatedProperty(
          NNStorage.DEPRECATED_MESSAGE_DIGEST_PROPERTY);
      if (md5 == null) {
        throw new InconsistentFSStateException(sdForProperties.getRoot(),
            "Message digest property " +
            NNStorage.DEPRECATED_MESSAGE_DIGEST_PROPERTY +
            " not set for storage directory " + sdForProperties.getRoot());
      }
      loadFSImage(imageFile.getFile(), new MD5Hash(md5), target, recovery);
    } else {
      // We don't have any record of the md5sum
      loadFSImage(imageFile.getFile(), null, target, recovery);
View Full Code Here

Examples of org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory

    try {
      List<Thread> saveThreads = new ArrayList<Thread>();
      // save images into current
      for (Iterator<StorageDirectory> it
             = storage.dirIterator(NameNodeDirType.IMAGE); it.hasNext();) {
        StorageDirectory sd = it.next();
        FSImageSaver saver = new FSImageSaver(ctx, sd);
        Thread saveThread = new Thread(saver, saver.toString());
        saveThreads.add(saveThread);
        saveThread.start();
      }
View Full Code Here

Examples of org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory

    numTransactions = totalTimeTransactions = 0;
    if (editStreams == null)
      editStreams = new ArrayList<EditLogOutputStream>();
    for (Iterator<StorageDirectory> it =
           fsimage.dirIterator(NameNodeDirType.EDITS); it.hasNext();) {
      StorageDirectory sd = it.next();
      File eFile = getEditFile(sd);
      try {
        EditLogOutputStream eStream = new EditLogFileOutputStream(eFile);
        editStreams.add(eStream);
      } catch (IOException e) {
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.