Package org.apache.hadoop.hdfs

Examples of org.apache.hadoop.hdfs.DistributedRaidFileSystem$ExtFSDataInputStream$ExtFsInputStream


      corruptBlock(srcFile, locations.get(blockNumToCorrupt).getBlock(),
            NUM_DATANODES, true);
    }

    // Validate
    DistributedRaidFileSystem raidfs = getRaidFS();
    assertTrue(validateFile(raidfs, srcFile, length, crc));
  }
View Full Code Here


                            UUID.randomUUID().toString());

    long crc = TestRaidDfs.createTestFilePartialLastBlock(fileSys, srcPath,
        1, blockNum, blockSize);
   
    DistributedRaidFileSystem raidFs = getRaidFS();
    assertTrue(raidFs.exists(srcPath));
   
    // generate the parity files.
    doRaid(srcPath, codec);

    FileStatus file1Stat = fileSys.getFileStatus(srcPath);
    long length = file1Stat.getLen();
    LocatedBlocks file1Loc =
        RaidDFSUtil.getBlockLocations((DistributedFileSystem)fileSys,
            srcPath.toUri().getPath(),
            0, length);
    // corrupt file1
   
    for (int idx: corruptBlockIdxs) {
      corruptBlock(file1Loc.get(idx).getBlock().getBlockName(),
                                dfs);
    }
    RaidDFSUtil.reportCorruptBlocks((DistributedFileSystem)fileSys, srcPath,
                         corruptBlockIdxs, blockSize);
   
    // verify the partial read
    byte[] buffer = new byte[bufferSize];
    FSDataInputStream in = raidFs.open(srcPath);
   
    long numRead = 0;
    CRC32 newcrc = new CRC32();
   
    int num = 0;
View Full Code Here

          new Path ("/user/dikang/raidtest/rename/f2"),
          new Path ("/user/dikang/raidtest/rename/f3")};
     
      Path destHarPath = new Path ("/destraid/user/dikang/raidtest/rename");
     
      DistributedRaidFileSystem raidFs = getRaidFS();
      for (Path srcPath : testPathList) {
        TestRaidDfs.createTestFilePartialLastBlock(fileSys, srcPath,
            1, 8, 8192L);
      }
     
      raidFs.mkdirs(destHarPath);
      raidFs.mkdirs(new Path(destHarPath, "rename" + RaidNode.HAR_SUFFIX));
     
      raidFs.rename(new Path("/user/dikang/raidtest"),
          new Path("/user/dikang/raidtest1"));
      fail("Expected fail for HAR rename");
    } catch (IOException ie) {
      String message = ie.getMessage();
      assertTrue(message.contains("HAR dir"));
View Full Code Here

     
      TestRaidDfs.createTestFilePartialLastBlock(fileSys, srcPath,
                            1, 8, 8192L);
      TestRaidDfs.createTestFilePartialLastBlock(fileSys, srcPath2,
                            1, 8, 8192L);
      DistributedRaidFileSystem raidFs = getRaidFS();
      assertTrue(raidFs.exists(srcPath));
      assertFalse(raidFs.exists(destPath));
      // generate the parity files.
      doRaid(srcPath, Codec.getCodec("xor"));
      doRaid(srcPath2, Codec.getCodec("xor"));
      ParityFilePair parity = ParityFilePair.getParityFile(
          Codec.getCodec("xor"),
          srcPath,
          conf);
      Path srcParityPath = parity.getPath();
      assertTrue(raidFs.exists(srcParityPath));
      parity = ParityFilePair.getParityFile(Codec.getCodec("xor"),
          destPath,
          conf);
      assertNull(parity);
      // do the rename file
      assertTrue(raidFs.rename(srcPath, destPath));
      // verify the results.
      assertFalse(raidFs.exists(srcPath));
      assertTrue(raidFs.exists(destPath));
      assertFalse(raidFs.exists(srcParityPath));
      parity = ParityFilePair.getParityFile(Codec.getCodec("xor"),
          destPath,
          conf);
      assertTrue(raidFs.exists(parity.getPath()));
     
      // rename the dir
      assertFalse(raidFs.exists(destDirPath));
      assertTrue(raidFs.rename(srcPath2.getParent(), destDirPath));
      // verify the results.
      assertFalse(raidFs.exists(srcPath2.getParent()));
      assertTrue(raidFs.exists(destDirPath));
     
      parity = ParityFilePair.getParityFile(Codec.getCodec("xor"),
          destPath2,
          conf);
      assertTrue(raidFs.exists(parity.getPath()));
    } finally {
      stopCluster();
    }
  }
View Full Code Here

      TestRaidDfs.createTestFilePartialLastBlock(fileSys, srcPath,
          1, 8, 8192L);
      TestRaidDfs.createTestFilePartialLastBlock(fileSys, srcPath2,
          1, 8, 8192L);
      DistributedRaidFileSystem raidFs = getRaidFS();
      assertTrue(raidFs.exists(srcPath));
      assertTrue(raidFs.exists(srcPath2));

      // generate the parity files.
      doRaid(srcPath, Codec.getCodec("xor"));
      doRaid(srcPath2, Codec.getCodec("xor"));
      ParityFilePair parity = ParityFilePair.getParityFile(
          Codec.getCodec("xor"),
          srcPath,
          conf);
      Path srcParityPath = parity.getPath();
      ParityFilePair parity2 = ParityFilePair.getParityFile(
          Codec.getCodec("xor"),
          srcPath2,
          conf);
      Path srcParityPath2 = parity2.getPath();
      assertTrue(raidFs.exists(srcParityPath));
      assertTrue(raidFs.exists(srcParityPath2));

      // do the delete file
      assertTrue(raidFs.delete(srcPath));

      // verify the results.
      assertFalse(raidFs.exists(srcPath));
      assertFalse(raidFs.exists(srcParityPath));
      assertTrue(raidFs.exists(srcParityPath2));

      // do the undelete using non-exist userName
      String nonExistedUser = UUID.randomUUID().toString();
      assertFalse(raidFs.undelete(srcPath, nonExistedUser));

      // verify the results
      assertFalse(raidFs.exists(srcPath));
      assertFalse(raidFs.exists(srcParityPath));
      assertTrue(raidFs.exists(srcParityPath2));

      // do the undelete file using current userName
      assertTrue(raidFs.undelete(srcPath, null));
      //verify the results.
      assertTrue(raidFs.exists(srcPath));
      assertTrue(raidFs.exists(srcParityPath));
      assertTrue(raidFs.exists(srcParityPath2));

      // delete the dir
      assertTrue(raidFs.delete(srcPath2.getParent()));
      // verify the results.
      assertFalse(raidFs.exists(srcPath2.getParent()));
      assertFalse(raidFs.exists(srcParityPath));
      assertFalse(raidFs.exists(srcParityPath2));
      assertFalse(raidFs.exists(srcParityPath.getParent()));

      // undelete the dir
      assertTrue(raidFs.undelete(srcPath2.getParent(), null));
      // verify the results.
      assertTrue(raidFs.exists(srcPath));
      assertTrue(raidFs.exists(srcPath2));
      assertTrue(raidFs.exists(srcParityPath));
      assertTrue(raidFs.exists(srcParityPath2));
    } finally {
      stopCluster();
    }
  }
View Full Code Here

      }
      affectedFiles.add(bi.fileIdx);
    }
    // validate files
    if (validate) {
      DistributedRaidFileSystem raidfs = getRaidFS(fileSys, conf);
      for (Integer fid: affectedFiles) {
        FileStatus stat = lfs.get(fid);
        assertTrue(TestRaidDfs.validateFile(raidfs, stat.getPath(),
            lengths[fid], crcs[fid]));
        // test readFully
        byte[] filebytes = new byte[(int)stat.getLen()];
        FSDataInputStream stm = raidfs.open(stat.getPath());
        stm.readFully(0, filebytes);
        CRC32 crc = new CRC32();
        crc.update(filebytes, 0, filebytes.length);
        assertEquals(crcs[fid], crc.getValue());
      }
View Full Code Here

                            UUID.randomUUID().toString());

    long crc = TestRaidDfs.createTestFilePartialLastBlock(fileSys, srcPath,
        1, blockNum, blockSize);
   
    DistributedRaidFileSystem raidFs = getRaidFS();
    assertTrue(raidFs.exists(srcPath));
   
    // generate the parity files.
    doRaid(srcPath, codec);

    FileStatus file1Stat = fileSys.getFileStatus(srcPath);
    long length = file1Stat.getLen();
    LocatedBlocks file1Loc =
        RaidDFSUtil.getBlockLocations((DistributedFileSystem)fileSys,
            srcPath.toUri().getPath(),
            0, length);
    // corrupt file1
   
    for (int idx: corruptBlockIdxs) {
      corruptBlock(file1Loc.get(idx).getBlock(),
                                dfs);
    }
    RaidDFSUtil.reportCorruptBlocks((DistributedFileSystem)fileSys, srcPath,
                         corruptBlockIdxs, blockSize);
   
    // verify the partial read
    byte[] buffer = new byte[bufferSize];
    FSDataInputStream in = raidFs.open(srcPath);
   
    long numRead = 0;
    CRC32 newcrc = new CRC32();
   
    int num = 0;
View Full Code Here

          new Path ("/user/dikang/raidtest/rename/f2"),
          new Path ("/user/dikang/raidtest/rename/f3")};
     
      Path destHarPath = new Path ("/destraid/user/dikang/raidtest/rename");
     
      DistributedRaidFileSystem raidFs = getRaidFS();
      for (Path srcPath : testPathList) {
        TestRaidDfs.createTestFilePartialLastBlock(fileSys, srcPath,
            1, 8, 8192L);
      }
     
      raidFs.mkdirs(destHarPath);
      raidFs.mkdirs(new Path(destHarPath, "rename" + RaidNode.HAR_SUFFIX));
     
      raidFs.rename(new Path("/user/dikang/raidtest"),
          new Path("/user/dikang/raidtest1"));
      fail("Expected fail for HAR rename");
    } catch (IOException ie) {
      String message = ie.getMessage();
      assertTrue(message.contains("HAR dir"));
View Full Code Here

     
      TestRaidDfs.createTestFilePartialLastBlock(fileSys, srcPath,
                            1, 8, 8192L);
      TestRaidDfs.createTestFilePartialLastBlock(fileSys, srcPath2,
                            1, 8, 8192L);
      DistributedRaidFileSystem raidFs = getRaidFS();
      assertTrue(raidFs.exists(srcPath));
      assertFalse(raidFs.exists(destPath));
      // generate the parity files.
      doRaid(srcPath, Codec.getCodec("xor"));
      doRaid(srcPath2, Codec.getCodec("xor"));
     
      FileStatus srcStat = fileSys.getFileStatus(srcPath);
      ParityFilePair parity = ParityFilePair.getParityFile(
          Codec.getCodec("xor"),
          srcStat,
          conf);
      Path srcParityPath = parity.getPath();
      assertTrue(raidFs.exists(srcParityPath));
      assertFalse(raidFs.exists(destPath));
     
      // do the rename file
      assertTrue(raidFs.rename(srcPath, destPath));
      // verify the results.
      assertFalse(raidFs.exists(srcPath));
      assertTrue(raidFs.exists(destPath));
      assertFalse(raidFs.exists(srcParityPath));
      FileStatus srcDest = fileSys.getFileStatus(destPath);
      parity = ParityFilePair.getParityFile(Codec.getCodec("xor"),
          srcDest,
          conf);
      assertTrue(raidFs.exists(parity.getPath()));
     
      // rename the dir
      assertFalse(raidFs.exists(destDirPath));
      assertTrue(raidFs.rename(srcPath2.getParent(), destDirPath));
      // verify the results.
      assertFalse(raidFs.exists(srcPath2.getParent()));
      assertTrue(raidFs.exists(destDirPath));
      FileStatus srcDest2 = fileSys.getFileStatus(destPath2);
      parity = ParityFilePair.getParityFile(Codec.getCodec("xor"),
          srcDest2,
          conf);
      assertTrue(raidFs.exists(parity.getPath()));
     
      // try to rename not existed file.
      Path notExistedPath = new Path("/user/dhruba/raidtest/raidnotexist");
      Path notExistedPath2 = new Path("/user/dhruba/raidtest/raidnotexist2");
      assertFalse(raidFs.rename(notExistedPath, notExistedPath2));
    } finally {
      stopCluster();
    }
  }
View Full Code Here

      Path srcPath2 = new Path("/tmp/raidtest/rename/f2");
      TestRaidDfs.createTestFilePartialLastBlock(fileSys, srcPath,
          1, 8, 8192L);
      TestRaidDfs.createTestFilePartialLastBlock(fileSys, srcPath2,
          1, 8, 8192L);
      DistributedRaidFileSystem raidFs = getRaidFS();
      assertTrue(raidFs.exists(srcPath));
      assertTrue(raidFs.exists(srcPath2));

      // generate the parity files.
      doRaid(srcPath, Codec.getCodec("xor"));
      doRaid(srcPath2, Codec.getCodec("xor"));
     
      FileStatus srcStat = fileSys.getFileStatus(srcPath);
      FileStatus srcStat2 = fileSys.getFileStatus(srcPath2);
      ParityFilePair parity = ParityFilePair.getParityFile(
          Codec.getCodec("xor"),
          srcStat,
          conf);
      Path srcParityPath = parity.getPath();
      ParityFilePair parity2 = ParityFilePair.getParityFile(
          Codec.getCodec("xor"),
          srcStat2,
          conf);
      Path srcParityPath2 = parity2.getPath();
      assertTrue(raidFs.exists(srcParityPath));
      assertTrue(raidFs.exists(srcParityPath2));
     
      // test delete file
      raidFs.delete(srcPath);
      // verify we did not delete the parityPath
      assertFalse(raidFs.exists(srcPath));
      assertTrue(raidFs.exists(srcParityPath));
     
      // test rename file
      raidFs.rename(srcPath2, new Path("/tmp/raidtest/rename/f3"));
      // verify we did not rename the parityPath
      assertFalse(raidFs.exists(srcPath2));
      assertTrue(raidFs.exists(srcParityPath2));
    } finally {
      stopCluster();
    }
  }
View Full Code Here

TOP

Related Classes of org.apache.hadoop.hdfs.DistributedRaidFileSystem$ExtFSDataInputStream$ExtFsInputStream

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.