Examples of FSDataInputStream

eu.stratosphere.core.fs.FSDataInputStream
Interface for a data input stream to a file on a {@link FileSystem}.
org.apache.flink.core.fs.FSDataInputStream
Interface for a data input stream to a file on a {@link FileSystem}.
org.apache.hadoop.fs.FSDataInputStream
Utility that wraps a {@link FSInputStream} in a {@link DataInputStream}and buffers input through a {@link BufferedInputStream}.

Examples of eu.stratosphere.core.fs.FSDataInputStream

      // Write out the length of the file
      final FileStatus file = fs.getFileStatus(jar);
      out.writeLong(file.getLen());


      // Now write the jar file
      final FSDataInputStream inStream = fs.open(this.userJars.get(i));
      final byte[] buf = new byte[BUFFERSIZE];
      int read = inStream.read(buf, 0, buf.length);
      while (read > 0) {
        out.write(buf, 0, read);
        read = inStream.read(buf, 0, buf.length);
      }
    }
  }

View Full Code Here

Examples of org.apache.flink.core.fs.FSDataInputStream

        final FileSystem fs = FileSystem.get(this.split.getPath().toUri());
        this.fdis = fs.open(this.split.getPath());
        
        // check for canceling and close the stream in that case, because no one will obtain it
        if (this.aborted) {
          final FSDataInputStream f = this.fdis;
          this.fdis = null;
          f.close();
        }
      }
      catch (Throwable t) {
        this.error = t;
      }

View Full Code Here

Examples of org.apache.hadoop.fs.FSDataInputStream

    FileStatus status = getTableInfoPath(fs, tableDir);
    if (status == null) {
      throw new TableInfoMissingException("No .tableinfo file under "
          + tableDir.toUri());
    }
    FSDataInputStream fsDataInputStream = fs.open(status.getPath());
    HTableDescriptor hTableDescriptor = null;
    try {
      hTableDescriptor = new HTableDescriptor();
      hTableDescriptor.readFields(fsDataInputStream);
    } finally {
      fsDataInputStream.close();
    }
    return new TableDescriptorModtime(status.getModificationTime(), hTableDescriptor);
  }

View Full Code Here

Examples of org.apache.hadoop.fs.FSDataInputStream

      return new Path(outputArchive, path);
    }


    private boolean copyFile(final Context context, final Path inputPath, final Path outputPath)
        throws IOException {
      FSDataInputStream in = openSourceFile(inputPath);
      if (in == null) {
        context.getCounter(Counter.MISSING_FILES).increment(1);
        return false;
      }


      try {
        // Verify if the input file exists
        FileStatus inputStat = getFileStatus(inputFs, inputPath);
        if (inputStat == null) return false;


        // Verify if the output file exists and is the same that we want to copy
        FileStatus outputStat = getFileStatus(outputFs, outputPath);
        if (outputStat != null && sameFile(inputStat, outputStat)) {
          LOG.info("Skip copy " + inputPath + " to " + outputPath + ", same file.");
          return true;
        }


        context.getCounter(Counter.BYTES_EXPECTED).increment(inputStat.getLen());


        // Ensure that the output folder is there and copy the file
        outputFs.mkdirs(outputPath.getParent());
        FSDataOutputStream out = outputFs.create(outputPath, true);
        try {
          if (!copyData(context, inputPath, in, outputPath, out, inputStat.getLen()))
            return false;
        } finally {
          out.close();
        }


        // Preserve attributes
        return preserveAttributes(outputPath, inputStat);
      } finally {
        in.close();
      }
    }

View Full Code Here

Examples of org.apache.hadoop.fs.FSDataInputStream

   */
  public static SnapshotDescription readSnapshotInfo(FileSystem fs, Path snapshotDir)
      throws CorruptedSnapshotException {
    Path snapshotInfo = new Path(snapshotDir, SNAPSHOTINFO_FILE);
    try {
      FSDataInputStream in = null;
      try {
        in = fs.open(snapshotInfo);
        return SnapshotDescription.parseFrom(in);
      } finally {
        if (in != null) in.close();
      }
    } catch (IOException e) {
      throw new CorruptedSnapshotException("Couldn't read snapshot info from:" + snapshotInfo, e);
    }
  }

View Full Code Here

Examples of org.apache.hadoop.fs.FSDataInputStream

    }


    Path regioninfo = new Path(regionDir, HRegion.REGIONINFO_FILE);
    FileSystem fs = regioninfo.getFileSystem(getConf());


    FSDataInputStream in = fs.open(regioninfo);
    HRegionInfo hri = new HRegionInfo();
    hri.readFields(in);
    in.close();
    LOG.debug("HRegionInfo read: " + hri.toString());
    hbi.hdfsEntry.hri = hri;
  }

View Full Code Here

Examples of org.apache.hadoop.fs.FSDataInputStream

    Path regionInfo = new Path(regionDir, HRegion.REGIONINFO_FILE);
    // make sure the file exists
    if (!fs.exists(regionInfo)) {
      throw new CorruptedSnapshotException("No region info found for region:" + region, snapshot);
    }
    FSDataInputStream in = fs.open(regionInfo);
    HRegionInfo found = new HRegionInfo();
    try {
      found.readFields(in);
      if (!region.equals(found)) {
        throw new CorruptedSnapshotException("Found region info (" + found
           + ") doesn't match expected region:" + region, snapshot);
      }
    } finally {
      in.close();
    }


    // make sure we have the expected recovered edits files
    TakeSnapshotUtils.verifyRecoveredEdits(fs, snapshotDir, found, snapshot);

View Full Code Here

Examples of org.apache.hadoop.fs.FSDataInputStream

  public static String getVersion(FileSystem fs, Path rootdir)
  throws IOException {
    Path versionFile = new Path(rootdir, HConstants.VERSION_FILE_NAME);
    String version = null;
    if (fs.exists(versionFile)) {
      FSDataInputStream s =
        fs.open(versionFile);
      try {
        version = DataInputStream.readUTF(s);
      } catch (EOFException eof) {
        LOG.warn("Version file was empty, odd, will try to set it.");
      } finally {
        s.close();
      }
    }
    return version;
  }

View Full Code Here

Examples of org.apache.hadoop.fs.FSDataInputStream

  public static String getClusterId(FileSystem fs, Path rootdir)
      throws IOException {
    Path idPath = new Path(rootdir, HConstants.CLUSTER_ID_FILE_NAME);
    String clusterId = null;
    if (fs.exists(idPath)) {
      FSDataInputStream in = fs.open(idPath);
      try {
        clusterId = in.readUTF();
      } catch (EOFException eof) {
        LOG.warn("Cluster ID file "+idPath.toString()+" was empty");
      } finally{
        in.close();
      }
    } else {
      LOG.warn("Cluster ID file does not exist at " + idPath.toString());
    }
    return clusterId;

View Full Code Here

Examples of org.apache.hadoop.fs.FSDataInputStream

      // if the snapshot is bad
      if (!fs.exists(info)) {
        LOG.error("Snapshot information for " + snapshot.getPath() + " doesn't exist");
        continue;
      }
      FSDataInputStream in = null;
      try {
        in = fs.open(info);
        SnapshotDescription desc = SnapshotDescription.parseFrom(in);
        snapshotDescs.add(desc);
      } catch (IOException e) {
        LOG.warn("Found a corrupted snapshot " + snapshot.getPath(), e);
      } finally {
        if (in != null) {
          in.close();
        }
      }
    }
    return snapshotDescs;
  }

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.