Examples of org.apache.hadoop.hbase.fs.HFileSystem

org.apache.hadoop.hbase.fs.HFileSystem
An encapsulation for the FileSystem object that hbase uses to access data. This class allows the flexibility of using separate filesystem objects for reading and writing hfiles and hlogs. In future, if we want to make hlogs be in a different filesystem, this is the place to make it happen.

  @Test
  public void testStoreFileScannerThrowsErrors() throws IOException {
    Path hfilePath = new Path(new Path(
        util.getDataTestDir("internalScannerExposesErrors"),
        "regionname"), "familyname");
    HFileSystem hfs = (HFileSystem)util.getTestFileSystem();
    FaultyFileSystem faultyfs = new FaultyFileSystem(hfs.getBackingFs());
    HFileSystem fs = new HFileSystem(faultyfs);
    CacheConfig cacheConf = new CacheConfig(util.getConfiguration());
    HFileContext meta = new HFileContextBuilder().withBlockSize(2 * 1024).build();
    StoreFile.Writer writer = new StoreFile.WriterBuilder(
        util.getConfiguration(), cacheConf, hfs)
            .withOutputDir(hfilePath)

View Full Code Here

        is.close();


        // Now, use a completely new reader. Switch off hbase checksums in 
        // the configuration. In this case, we should not detect
        // any retries within hbase. 
        HFileSystem newfs = new HFileSystem(TEST_UTIL.getConfiguration(), false);
        assertEquals(false, newfs.useHBaseChecksum());
        is = new FSDataInputStreamWrapper(newfs, path);
        hbr = new FSReaderV2Test(is, totalSize, newfs, path, meta);
        b = hbr.readBlockData(0, -1, -1, pread);
        is.close();
        b.sanityCheck();

View Full Code Here

   * Tests that log rolling doesn't hang when no data is written.
   */
  @Test(timeout=120000)
  public void testLogRollOnNothingWritten() throws Exception {
    Configuration conf = TEST_UTIL.getConfiguration();
    HFileSystem fs = new HFileSystem(conf, false);
    HLog newLog = HLogFactory.createHLog(fs.getBackingFs(),
      FSUtils.getRootDir(conf), "test", conf, null, "test.com:8080:1");
    try {
      // Now roll the log before we write anything.
      newLog.rollWriter(true);
    } finally {

View Full Code Here

      // to defaults).
      FSUtils.setFsDefault(this.conf, FSUtils.getRootDir(this.conf));
      // Get fs instance used by this RS.  Do we use checksum verification in the hbase? If hbase
      // checksum verification enabled, then automatically switch off hdfs checksum verification.
      boolean useHBaseChecksum = conf.getBoolean(HConstants.HBASE_CHECKSUM_VERIFICATION, true);
      this.fs = new HFileSystem(this.conf, useHBaseChecksum);
      this.rootDir = FSUtils.getRootDir(this.conf);
      this.tableDescriptors = new FSTableDescriptors(this.fs, this.rootDir, true);
      this.hlog = setupWALAndReplication();
      // Init in here rather than in constructor after thread name has been set
      this.metricsRegionServer = new MetricsRegionServer(new MetricsRegionServerWrapperImpl(this));

View Full Code Here

   * @throws IOException If file is invalid, will throw CorruptHFileException flavored IOException
   */
  public static Reader createReader(FileSystem fs, Path path,
      FSDataInputStreamWrapper fsdis, long size, CacheConfig cacheConf, Configuration conf)
      throws IOException {
    HFileSystem hfs = null;


    // If the fs is not an instance of HFileSystem, then create an
    // instance of HFileSystem that wraps over the specified fs.
    // In this case, we will not be able to avoid checksumming inside
    // the filesystem.
    if (!(fs instanceof HFileSystem)) {
      hfs = new HFileSystem(fs);
    } else {
      hfs = (HFileSystem)fs;
    }
    return pickReaderVersion(path, fsdis, size, cacheConf, hfs, conf);
  }

View Full Code Here

   * Tests that log rolling doesn't hang when no data is written.
   */
  @Test(timeout=120000)
  public void testLogRollOnNothingWritten() throws Exception {
    Configuration conf = TEST_UTIL.getConfiguration();
    HFileSystem fs = new HFileSystem(conf, false);
    HLog newLog = HLogFactory.createHLog(fs.getBackingFs(),
      FSUtils.getRootDir(conf), "test", conf, null, "test.com:8080:1");
    try {
      // Now roll the log before we write anything.
      newLog.rollWriter(true);
    } finally {

View Full Code Here

  @Test
  public void testHFileScannerThrowsErrors() throws IOException {
    Path hfilePath = new Path(new Path(
        util.getDataTestDir("internalScannerExposesErrors"),
        "regionname"), "familyname");
    HFileSystem hfs = (HFileSystem)util.getTestFileSystem();
    FaultyFileSystem faultyfs = new FaultyFileSystem(hfs.getBackingFs());
    FileSystem fs = new HFileSystem(faultyfs);
    CacheConfig cacheConf = new CacheConfig(util.getConfiguration());
    HFileContext meta = new HFileContextBuilder().withBlockSize(2 * 1024).build();
    StoreFile.Writer writer = new StoreFile.WriterBuilder(
        util.getConfiguration(), cacheConf, hfs)
            .withOutputDir(hfilePath)

View Full Code Here

  @Test
  public void testStoreFileScannerThrowsErrors() throws IOException {
    Path hfilePath = new Path(new Path(
        util.getDataTestDir("internalScannerExposesErrors"),
        "regionname"), "familyname");
    HFileSystem hfs = (HFileSystem)util.getTestFileSystem();
    FaultyFileSystem faultyfs = new FaultyFileSystem(hfs.getBackingFs());
    HFileSystem fs = new HFileSystem(faultyfs);
    CacheConfig cacheConf = new CacheConfig(util.getConfiguration());
    HFileContext meta = new HFileContextBuilder().withBlockSize(2 * 1024).build();
    StoreFile.Writer writer = new StoreFile.WriterBuilder(
        util.getConfiguration(), cacheConf, hfs)
            .withOutputDir(hfilePath)

View Full Code Here

   * @throws IOException If file is invalid, will throw CorruptHFileException flavored IOException
   */
  public static Reader createReader(FileSystem fs, Path path,
      FSDataInputStreamWrapper fsdis, long size, CacheConfig cacheConf, Configuration conf)
      throws IOException {
    HFileSystem hfs = null;


    // If the fs is not an instance of HFileSystem, then create an
    // instance of HFileSystem that wraps over the specified fs.
    // In this case, we will not be able to avoid checksumming inside
    // the filesystem.
    if (!(fs instanceof HFileSystem)) {
      hfs = new HFileSystem(fs);
    } else {
      hfs = (HFileSystem)fs;
    }
    return pickReaderVersion(path, fsdis, size, cacheConf, hfs, conf);
  }

View Full Code Here

        is.close();


        // Now, use a completely new reader. Switch off hbase checksums in 
        // the configuration. In this case, we should not detect
        // any retries within hbase. 
        HFileSystem newfs = new HFileSystem(TEST_UTIL.getConfiguration(), false);
        assertEquals(false, newfs.useHBaseChecksum());
        is = new FSDataInputStreamWrapper(newfs, path);
        hbr = new FSReaderImplTest(is, totalSize, newfs, path, meta);
        b = hbr.readBlockData(0, -1, -1, pread);
        is.close();
        b.sanityCheck();

View Full Code Here

0 1 2 3 4 5 6 7 8

TOP

Related Classes of org.apache.hadoop.hbase.fs.HFileSystem

org.apache.hadoop.conf.Configuration

org.apache.hadoop.fs.FileSystem

org.apache.hadoop.hbase.io.FSDataInputStreamWrapper

org.apache.hadoop.hbase.io.hfile.HFile

org.apache.hadoop.hbase.io.hfile.TestChecksum

org.apache.hadoop.hbase.regionserver.HRegionServer

org.apache.hadoop.hbase.regionserver.TestFSErrorsExposed

org.apache.hadoop.hbase.regionserver.wal.TestLogRolling

org.apache.hadoop.hbase.ServerName

org.apache.hadoop.hdfs.DFSClient

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.