Examples of LostFileInfo


Examples of org.apache.hadoop.raid.DistBlockIntegrityMonitor.Worker.LostFileInfo

        // Remove the RAID prefix to get the source dir.
        srcDirsToWatchOutFor.add(
            parentUriPath.substring(parentUriPath.indexOf(Path.SEPARATOR, 1)));
        int numCorrupt = corruptFiles.get(p);
        Priority priority = (numCorrupt > 1) ? Priority.HIGH : Priority.LOW;
        LostFileInfo fileInfo = fileIndex.get(p);
        if (fileInfo == null || priority.higherThan(fileInfo.getHighestPriority())) {
          fileToPriority.put(p, priority);
        }
      }
      // Loop over src files now.
      for (Iterator<String> it = corruptFiles.keySet().iterator(); it.hasNext(); ) {
        String p = it.next();
        if (BlockIntegrityMonitor.isSourceFile(p)) {
          FileStatus stat = fs.getFileStatus(new Path(p));
          if (stat.getReplication() >= notRaidedReplication) {
            continue;
          }
          if (BlockIntegrityMonitor.doesParityDirExist(fs, p)) {
            int numCorrupt = corruptFiles.get(p);
            Priority priority = Priority.LOW;
            if (stat.getReplication() > 1) {
              // If we have a missing block when replication > 1, it is high pri.
              priority = Priority.HIGH;
            } else {
              // Replication == 1. Assume Reed Solomon parity exists.
              // If we have more than one missing block when replication == 1, then
              // high pri.
              priority = (numCorrupt > 1) ? Priority.HIGH : Priority.LOW;
            }
            // If priority is low, check if the scan of corrupt parity files found
            // the src dir to be risky.
            if (priority == Priority.LOW) {
              Path parent = new Path(p).getParent();
              String parentUriPath = parent.toUri().getPath();
              if (srcDirsToWatchOutFor.contains(parentUriPath)) {
                priority = Priority.HIGH;
              }
            }
            LostFileInfo fileInfo = fileIndex.get(p);
            if (fileInfo == null || priority.higherThan(fileInfo.getHighestPriority())) {
              fileToPriority.put(p, priority);
            }
          }
        }
      }
View Full Code Here

Examples of org.apache.hadoop.raid.DistBlockIntegrityMonitor.Worker.LostFileInfo

        // Files with more than 4 blocks being decommissioned get a bump.
        // Otherwise, copying jobs have the lowest priority.
        Priority priority = ((decommissioningFiles.get(file) > Codec.getCodec("rs").parityLength) ?
            Priority.LOW : Priority.LOWEST);

        LostFileInfo fileInfo = fileIndex.get(file);
        if (fileInfo == null || priority.higherThan(fileInfo.getHighestPriority())) {
          fileToPriority.put(file, priority);
        }
      }
      return fileToPriority;
    }
View Full Code Here

Examples of org.apache.hadoop.raid.DistBlockIntegrityMonitor.Worker.LostFileInfo

    private List<LostFileInfo> updateFileIndex(
        String jobName, List<String> lostFiles, Priority priority) {
      List<LostFileInfo> fileInfos = new ArrayList<LostFileInfo>();

      for (String file: lostFiles) {
        LostFileInfo fileInfo = fileIndex.get(file);
        if (fileInfo != null) {
          fileInfo.addJob(jobName, priority);
        } else {
          fileInfo = new LostFileInfo(file, jobName, priority);
          fileIndex.put(file, fileInfo);
        }
        fileInfos.add(fileInfo);
      }
      return fileInfos;
View Full Code Here

Examples of org.apache.hadoop.raid.DistBlockIntegrityMonitor.Worker.LostFileInfo

      List<JobStatus> jobs = new ArrayList<JobStatus>();
      List<JobStatus> simFailJobs = new ArrayList<JobStatus>();
      List<String> highPriorityFileNames = new ArrayList<String>();
      for (Map.Entry<String, LostFileInfo> e : fileIndex.entrySet()) {
        String fileName = e.getKey();
        LostFileInfo fileInfo = e.getValue();
        Priority pri = fileInfo.getHighestPriority();
        if (pri == Priority.HIGH) {
          highPriorityFileNames.add(fileName);
          highPriorityFiles++;
        } else if (pri == Priority.LOW){
          lowPriorityFiles++;
View Full Code Here

Examples of org.apache.hadoop.raid.DistBlockIntegrityMonitor.Worker.LostFileInfo

            (failed ? " failed in " : " succeeded in ") +
            jobName);
        if (jobNames.isEmpty()) {
          // All jobs dealing with this file are done,
          // remove this file from the index
          LostFileInfo removed = fileIndex.remove(file);
          if (removed == null) {
            LOG.error("trying to remove file not in file index: " + file);
          }
          done = true;
        }
View Full Code Here

Examples of org.apache.hadoop.raid.DistBlockIntegrityMonitor.Worker.LostFileInfo

    private List<LostFileInfo> updateFileIndex(
        String jobName, List<String> lostFiles, Priority priority) {
      List<LostFileInfo> fileInfos = new ArrayList<LostFileInfo>();

      for (String file: lostFiles) {
        LostFileInfo fileInfo = fileIndex.get(file);
        if (fileInfo != null) {
          fileInfo.addJob(jobName, priority);
        } else {
          fileInfo = new LostFileInfo(file, jobName, priority);
          fileIndex.put(file, fileInfo);
        }
        fileInfos.add(fileInfo);
      }
      return fileInfos;
View Full Code Here

Examples of org.apache.hadoop.raid.DistBlockIntegrityMonitor.Worker.LostFileInfo

      List<JobStatus> failJobs = new ArrayList<JobStatus>();
      List<JobStatus> simFailJobs = new ArrayList<JobStatus>();
      List<String> highPriorityFileNames = new ArrayList<String>();
      for (Map.Entry<String, LostFileInfo> e : fileIndex.entrySet()) {
        String fileName = e.getKey();
        LostFileInfo fileInfo = e.getValue();
        Priority pri = fileInfo.getHighestPriority();
        if (pri == Priority.HIGH) {
          highPriorityFileNames.add(fileName);
          highPriorityFiles++;
        } else if (pri == Priority.LOW){
          lowPriorityFiles++;
View Full Code Here

Examples of org.apache.hadoop.raid.DistBlockIntegrityMonitor.Worker.LostFileInfo

            (failed ? " failed in " : " succeeded in ") +
            jobName);
        if (jobNames.isEmpty()) {
          // All jobs dealing with this file are done,
          // remove this file from the index
          LostFileInfo removed = fileIndex.remove(file);
          if (removed == null) {
            LOG.error("trying to remove file not in file index: " + file);
          }
          done = true;
        }
View Full Code Here

Examples of org.apache.hadoop.raid.DistBlockIntegrityMonitor.Worker.LostFileInfo

                // Replication == 1. Assume Reed Solomon parity exists.
                // If we have more than one missing block when replication == 1, then
                // high pri.
                priority = (corruptFile.numCorrupt > 1) ? Priority.HIGH : Priority.LOW;
              }
              LostFileInfo fileInfo = fileIndex.get(corruptFile.path);
              if (fileInfo == null || priority.higherThan(
                  fileInfo.getHighestPriority())) {
                addJobTime = addToJobFilesMap(jobFilesMap, priority,
                    corruptFile.path, detectTime);
              }
            }
          } else {
            // Dikang: for parity files, we use the total numbers for now.
            Priority priority = (corruptFile.numCorrupt > 1) ?
                Priority.HIGH : (codec.parityLength == 1)? Priority.HIGH: Priority.LOW;
            LostFileInfo fileInfo = fileIndex.get(corruptFile.path);
            if (fileInfo == null || priority.higherThan(
                fileInfo.getHighestPriority())) {
              addJobTime = addToJobFilesMap(jobFilesMap, priority, corruptFile.path,
                  detectTime);
            }
          }
          boolean isFileCorrupt = RaidShell.isFileCorrupt((DistributedFileSystem)fs,
View Full Code Here

Examples of org.apache.hadoop.raid.DistBlockIntegrityMonitor.Worker.LostFileInfo

        // Otherwise, copying jobs have the lowest priority.
        Priority priority = ((decommissioningFiles.get(file)
            > Codec.getCodec("rs").parityLength) ?
            Priority.LOW : Priority.LOWEST);

        LostFileInfo fileInfo = fileIndex.get(file);
        if (fileInfo == null || priority.higherThan(fileInfo.getHighestPriority())) {
          fileToPriority.put(file, priority);
        }
      }
      LOG.info("Found " + fileToPriority.size() + " new lost files");
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.