Examples of org.apache.hadoop.fs.FileSystem.open()

org.apache.hadoop.fs.FileSystem.open()
Opens an FSDataInputStream at the indicated Path. @param f the file name to open @param bufferSize the size of the buffer to be used.

            FileSplit s = (FileSplit) split;
            this.size = s.getLength();
            Path path = s.getPath();
            FileSystem fs = path.getFileSystem(context.getConfiguration());
            int blocks = computeBlocks(s);
            FSDataInputStream stream = fs.open(path);
            boolean succeed = false;
            try {
                if (s.getStart() != 0) {
                    assert s.getStart() % TemporaryFile.BLOCK_SIZE == 0;
                    stream.seek(s.getStart());

View Full Code Here

                LOG.debug(MessageFormat.format(
                        "reading remote cache checksum: {0}",
                        cacheFilePath));
            }
            long other;
            FSDataInputStream input = fs.open(cacheChecksumPath);
            try {
                other = input.readLong();
            } finally {
                input.close();
            }

View Full Code Here

    /*
     * Create a new file and read the block. The block should be marked 
     * verified since the client reads the block and verifies checksum. 
     */
    DFSTestUtil.createFile(fs, file2, 10, (short)1, 0);
    IOUtils.copyBytes(fs.open(file2), new IOUtils.NullOutputStream(), 
                      conf, true); 
    assertTrue(waitForVerification(dn, fs, file2) > startTime);
    
    cluster.shutdown();
  }

View Full Code Here

    corruptReplica(block, 1);
    corruptReplica(block, 2);


    // Read the file to trigger reportBadBlocks by client
    try {
      IOUtils.copyBytes(fs.open(file1), new IOUtils.NullOutputStream(), 
                        conf, true);
    } catch (IOException e) {
      // Ignore exception
    }

View Full Code Here

    // Loop until all corrupt replicas are reported
    int corruptReplicaSize = cluster.getNameNode().namesystem.
                              corruptReplicas.numCorruptReplicas(blk);
    while (corruptReplicaSize != numCorruptReplicas) {
      try {
        IOUtils.copyBytes(fs.open(file1), new IOUtils.NullOutputStream(), 
                          conf, true);
      } catch (IOException e) {
      }
      try {
        LOG.info("Looping until expected " + numCorruptReplicas + " are " +

View Full Code Here

    }


    Path regioninfo = new Path(regionDir, HRegion.REGIONINFO_FILE);
    FileSystem fs = regioninfo.getFileSystem(conf);


    FSDataInputStream in = fs.open(regioninfo);
    HRegionInfo hri = new HRegionInfo();
    hri.readFields(in);
    in.close();
    LOG.debug("HRegionInfo read: " + hri.toString());
    hbi.hdfsEntry.hri = hri;

View Full Code Here

      FileSystem.closeAll();
      FileSystem raidfs = FileSystem.get(dfsUri, clientConf);


      FileStatus stat = raidfs.getFileStatus(file);
      byte[] filebytes = new byte[(int)stat.getLen()];
      FSDataInputStream stm = raidfs.open(file);
      // Test that readFully returns.
      stm.readFully(filebytes, 0, (int)stat.getLen());


      stm = raidfs.open(file);
      // Test that readFully returns.

View Full Code Here

      byte[] filebytes = new byte[(int)stat.getLen()];
      FSDataInputStream stm = raidfs.open(file);
      // Test that readFully returns.
      stm.readFully(filebytes, 0, (int)stat.getLen());


      stm = raidfs.open(file);
      // Test that readFully returns.
      stm.readFully(filebytes);
    } finally {
      myTearDown();
    }

View Full Code Here

        FSDataOutputStream failureWriter = null;
        DataInputStream failureReader = null;
        try {
          failureWriter = fs.create(failurePath.suffix("/"
              + importStatus.getPath().getName()));
          failureReader = fs.open(importStatus.getPath());
          int read = 0;
          byte[] buffer = new byte[1024];
          while (-1 != (read = failureReader.read(buffer))) {
            failureWriter.write(buffer, 0, read);
          }

View Full Code Here

  public void testAuditAllowed() throws Exception {
    final Path file = new Path(fnames[0]);
    FileSystem userfs = DFSTestUtil.getFileSystemAs(userGroupInfo, conf);


    setupAuditLogs();
    InputStream istream = userfs.open(file);
    int val = istream.read();
    istream.close();
    verifyAuditLogs(true);
    assertTrue("failed to read from file", val >= 0);
  }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.