Examples of org.apache.hadoop.fs.HarStatus

org.apache.hadoop.fs.HarStatus
Class representing item (dir/file) in hadoop archive. {@see HarFileSystem} for description of the format used in index files

    if (!localFS.getFileStatus(local).isDir()) {
      throw new IOException("Path " + local + " is not a directory");
    }
    try {
      while (harReader.hasNext()) {
        HarStatus harStatus = harReader.getNext();
        String harPath = harStatus.getName();
        
        // skip top level dir
        if (harPath.equals(Path.SEPARATOR)) {
          continue;
        }
        String relativePath = harPath.substring(1);
        Path output = new Path(local, relativePath);
        if (harStatus.isDir()) {
          localFS.mkdirs(output);
        } else {
          OutputStream outputStream = null;
          FSDataInputStream inputStream = null;
          
          try {
            outputStream = localFS.create(output);
            Path partFile = new Path(archivePath, harStatus.getPartName());
            inputStream = new HarFSDataInputStream(fs, partFile,
                harStatus.getStartIndex(), harStatus.getLength(), conf.getInt("io.file.buffer.size", 4096));
            IOUtils.copyBytes(inputStream, outputStream, conf);
          } finally {
            if (outputStream != null) {
              outputStream.close();
            }

View Full Code Here

    public void map(LongWritable key, Text value, OutputCollector<IntWritable, Text> out,
        Reporter reporter) throws IOException {
      reporter.setStatus("Passing file " + value + " to archive.");
      reporter.progress();


      HarStatus harStatus = new HarStatus(value.toString());
      int hash = HarFileSystem.getHarHash(harStatus.getName());
      out.collect(new IntWritable(hash), value);
    }

View Full Code Here

      
      // merge the children of the same directories
      Map<String, HarStatus> harItems = new HashMap<String, HarStatus>();
      while(values.hasNext()) {
        Text value = values.next();
        HarStatus harStatus = new HarStatus(value.toString());
        if (harItems.containsKey(harStatus.getName())) {
          if (!harStatus.isDir()) {
            throw new RuntimeException("File " + harStatus.getName() + " already exists in har");
          }
          HarStatus existingHarStatus = harItems.get(harStatus.getName());
          existingHarStatus.getChildren().addAll(harStatus.getChildren());
        } else {
          harItems.put(harStatus.getName(), harStatus);
        }
      }

View Full Code Here

TOP

Related Classes of org.apache.hadoop.fs.HarStatus

org.apache.hadoop.tools.HadoopArchives

org.apache.hadoop.tools.HadoopArchives$HArchivesConvertingMapper

org.apache.hadoop.tools.HadoopArchives$HArchivesMergingReducer

java.io.IOException

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.