Examples of StoppableImplementation

org.apache.hadoop.hbase.util.StoppableImplementation
A base implementation for a Stoppable service

Examples of org.apache.hadoop.hbase.util.StoppableImplementation

    Mockito.reset(spy);
  }


  @Test
  public void testStoppedCleanerDoesNotDeleteFiles() throws Exception {
    Stoppable stop = new StoppableImplementation();
    Configuration conf = UTIL.getConfiguration();
    Path testDir = UTIL.getDataTestDir();
    FileSystem fs = UTIL.getTestFileSystem();
    String confKey = "hbase.test.cleaner.delegates";
    conf.set(confKey, AlwaysDelete.class.getName());


    AllValidPaths chore = new AllValidPaths("test-file-cleaner", stop, conf, fs, testDir, confKey);


    // also create a file in the top level directory
    Path topFile = new Path(testDir, "topFile");
    fs.create(topFile).close();
    assertTrue("Test file didn't get created.", fs.exists(topFile));


    // stop the chore
    stop.stop("testing stop");


    // run the chore
    chore.chore();


    // test that the file still exists

View Full Code Here

Examples of org.apache.hadoop.hbase.util.StoppableImplementation

   * another file added, in which case the directory shouldn't be deleted.
   * @throws IOException on failure
   */
  @Test
  public void testCleanerDoesNotDeleteDirectoryWithLateAddedFiles() throws IOException {
    Stoppable stop = new StoppableImplementation();
    Configuration conf = UTIL.getConfiguration();
    final Path testDir = UTIL.getDataTestDir();
    final FileSystem fs = UTIL.getTestFileSystem();
    String confKey = "hbase.test.cleaner.delegates";
    conf.set(confKey, AlwaysDelete.class.getName());

View Full Code Here

Examples of org.apache.hadoop.hbase.util.StoppableImplementation

   * This was from HBASE-7465.
   * @throws Exception on failure
   */
  @Test
  public void testNoExceptionFromDirectoryWithRacyChildren() throws Exception {
    Stoppable stop = new StoppableImplementation();
    // need to use a localutil to not break the rest of the test that runs on the local FS, which
    // gets hosed when we start to use a minicluster.
    HBaseTestingUtility localUtil = new HBaseTestingUtility();
    Configuration conf = localUtil.getConfiguration();
    final Path testDir = UTIL.getDataTestDir();

View Full Code Here

Examples of org.apache.hadoop.hbase.util.StoppableImplementation

      LOG.error(errorMsg);
      fail(errorMsg);
    }


    int familyValues = admin.getTableDescriptor(tableName).getFamily(family).getValues().size();
    StoppableImplementation stopper = new StoppableImplementation();


    //alter table every 10 sec
    Chore alterThread = new Chore("Alter Chore", 10000, stopper) {
      @Override
      protected void chore() {
        Random random = new Random();
        try {
          HTableDescriptor htd = admin.getTableDescriptor(tableName);
          String val = String.valueOf(random.nextInt());
          htd.getFamily(family).setValue(val, val);
          desc.getFamily(family).setValue(val, val); // save it for later
                                                     // control
          admin.modifyTable(tableName, htd);
        } catch (Exception ex) {
          LOG.warn("Caught exception", ex);
          fail(ex.getMessage());
        }
      }
    };


    //split table every 5 sec
    Chore splitThread = new Chore("Split thread", 5000, stopper) {
      @Override
      public void chore() {
        try {
          HRegion region = TEST_UTIL.getSplittableRegion(tableName, -1);
          if (region != null) {
            byte[] regionName = region.getRegionName();
            admin.flush(regionName);
            admin.compact(regionName);
            admin.split(regionName);
          } else {
            LOG.warn("Could not find suitable region for the table.  Possibly the " +
              "region got closed and the attempts got over before " +
              "the region could have got reassigned.");
          }
        } catch (NotServingRegionException nsre) {
          // the region may be in transition
          LOG.warn("Caught exception", nsre);
        } catch (Exception ex) {
          LOG.warn("Caught exception", ex);
          fail(ex.getMessage());
        }
      }
    };


    alterThread.start();
    splitThread.start();
    while (true) {
      List<HRegionInfo> regions = admin.getTableRegions(tableName);
      LOG.info(String.format("Table #regions: %d regions: %s:", regions.size(), regions));
      assertEquals(admin.getTableDescriptor(tableName), desc);
      for (HRegion region : TEST_UTIL.getMiniHBaseCluster().getRegions(tableName)) {
        assertEquals(desc, region.getTableDesc());
      }
      if (regions.size() >= 5) {
        break;
      }
      Threads.sleep(1000);
    }
    stopper.stop("test finished");


    int newFamilyValues = admin.getTableDescriptor(tableName).getFamily(family).getValues().size();
    LOG.info(String.format("Altered the table %d times", newFamilyValues - familyValues));
    assertTrue(newFamilyValues > familyValues); // at least one alter went
                                                // through

View Full Code Here

Examples of org.apache.hadoop.hbase.util.StoppableImplementation


    //SplitTransaction will update the meta table by offlining the parent region, and adding info
    //for daughters.
    HTable table = TEST_UTIL.createTable(TABLENAME, FAMILY);


    Stoppable stopper = new StoppableImplementation();
    RegionSplitter regionSplitter = new RegionSplitter(table);
    RegionChecker regionChecker = new RegionChecker(conf, stopper, TABLENAME);


    regionChecker.start();
    regionSplitter.start();


    //wait until the splitter is finished
    regionSplitter.join();
    stopper.stop(null);


    if (regionChecker.ex != null) {
      throw regionChecker.ex;
    }

View Full Code Here

Examples of org.apache.hadoop.hbase.util.StoppableImplementation

    Path familyDir = new Path(regionDir, "cf");


    Path sourceRegionDir = new Path(rootDir, regionDir);
    fs.mkdirs(sourceRegionDir);


    Stoppable stoppable = new StoppableImplementation();


    // The cleaner should be looping without long pauses to reproduce the race condition.
    HFileCleaner cleaner = new HFileCleaner(1, stoppable, conf, fs, archiveDir);
    try {
      cleaner.start();


      // Keep creating/archiving new files while the cleaner is running in the other thread
      long startTime = System.currentTimeMillis();
      for (long fid = 0; (System.currentTimeMillis() - startTime) < TEST_TIME; ++fid) {
        Path file = new Path(familyDir,  String.valueOf(fid));
        Path sourceFile = new Path(rootDir, file);
        Path archiveFile = new Path(archiveDir, file);


        fs.createNewFile(sourceFile);


        try {
          // Try to archive the file
          HFileArchiver.archiveRegion(fs, rootDir,
              sourceRegionDir.getParent(), sourceRegionDir);


          // The archiver succeded, the file is no longer in the original location
          // but it's in the archive location.
          LOG.debug("hfile=" + fid + " should be in the archive");
          assertTrue(fs.exists(archiveFile));
          assertFalse(fs.exists(sourceFile));
        } catch (IOException e) {
          // The archiver is unable to archive the file. Probably HBASE-7643 race condition.
          // in this case, the file should not be archived, and we should have the file
          // in the original location.
          LOG.debug("hfile=" + fid + " should be in the source location");
          assertFalse(fs.exists(archiveFile));
          assertTrue(fs.exists(sourceFile));


          // Avoid to have this file in the next run
          fs.delete(sourceFile, false);
        }
      }
    } finally {
      stoppable.stop("test end");
      cleaner.join();
      fs.delete(rootDir, true);
    }
  }

View Full Code Here

Examples of org.apache.hadoop.hbase.util.StoppableImplementation

    toCleanup.add(archiveDir);
    toCleanup.add(tableDir);


    Configuration conf = UTIL.getConfiguration();
    // setup the delegate
    Stoppable stop = new StoppableImplementation();
    HFileCleaner cleaner = setupAndCreateCleaner(conf, fs, archiveDir, stop);
    List<BaseHFileCleanerDelegate> cleaners = turnOnArchiving(STRING_TABLE_NAME, cleaner);
    final LongTermArchivingHFileCleaner delegate = (LongTermArchivingHFileCleaner) cleaners.get(0);


    // create the region

View Full Code Here

Examples of org.apache.hadoop.hbase.util.StoppableImplementation

    toCleanup.add(archiveDir);
    toCleanup.add(tableDir);
    toCleanup.add(otherTableDir);
    Configuration conf = UTIL.getConfiguration();
    // setup the delegate
    Stoppable stop = new StoppableImplementation();
    HFileCleaner cleaner = setupAndCreateCleaner(conf, fs, archiveDir, stop);
    List<BaseHFileCleanerDelegate> cleaners = turnOnArchiving(STRING_TABLE_NAME, cleaner);
    final LongTermArchivingHFileCleaner delegate = (LongTermArchivingHFileCleaner) cleaners.get(0);


    // create the region
    HColumnDescriptor hcd = new HColumnDescriptor(TEST_FAM);
    HRegion region = UTIL.createTestRegion(STRING_TABLE_NAME, hcd);
    loadFlushAndCompact(region, TEST_FAM);


    // create the another table that we don't archive
    hcd = new HColumnDescriptor(TEST_FAM);
    HRegion otherRegion = UTIL.createTestRegion(otherTable, hcd);
    loadFlushAndCompact(otherRegion, TEST_FAM);


    // get the current hfiles in the archive directory
    List<Path> files = getAllFiles(fs, archiveDir);
    if (files == null) {
      FSUtils.logFileSystemState(fs, archiveDir, LOG);
      throw new RuntimeException("Didn't load archive any files!");
    }


    // make sure we have files from both tables
    int initialCountForPrimary = 0;
    int initialCountForOtherTable = 0;
    for (Path file : files) {
      String tableName = file.getParent().getParent().getParent().getName();
      // check to which table this file belongs
      if (tableName.equals(otherTable)) initialCountForOtherTable++;
      else if (tableName.equals(STRING_TABLE_NAME)) initialCountForPrimary++;
    }


    assertTrue("Didn't archive files for:" + STRING_TABLE_NAME, initialCountForPrimary > 0);
    assertTrue("Didn't archive files for:" + otherTable, initialCountForOtherTable > 0);


    // run the cleaners, checking for each of the directories + files (both should be deleted and
    // need to be checked) in 'otherTable' and the files (which should be retained) in the 'table'
    CountDownLatch finished = setupCleanerWatching(delegate, cleaners, files.size() + 3);
    // run the cleaner
    cleaner.start();
    // wait for the cleaner to check all the files
    finished.await();
    // stop the cleaner
    stop.stop("");


    // know the cleaner ran, so now check all the files again to make sure they are still there
    List<Path> archivedFiles = getAllFiles(fs, archiveDir);
    int archivedForPrimary = 0;
    for(Path file: archivedFiles) {

View Full Code Here

Examples of org.apache.hadoop.hbase.util.StoppableImplementation

}




  @Test
  public void testSavesFilesOnRequest() throws Exception {
    Stoppable stop = new StoppableImplementation();
    Configuration conf = UTIL.getConfiguration();
    Path testDir = UTIL.getDataTestDir();
    FileSystem fs = UTIL.getTestFileSystem();
    String confKey = "hbase.test.cleaner.delegates";
    conf.set(confKey, NeverDelete.class.getName());

View Full Code Here

Examples of org.apache.hadoop.hbase.util.StoppableImplementation

    assertTrue("Empty directory didn't get deleted", fs.exists(parent));
  }


  @Test
  public void testDeletesEmptyDirectories() throws Exception {
    Stoppable stop = new StoppableImplementation();
    Configuration conf = UTIL.getConfiguration();
    Path testDir = UTIL.getDataTestDir();
    FileSystem fs = UTIL.getTestFileSystem();
    String confKey = "hbase.test.cleaner.delegates";
    conf.set(confKey, AlwaysDelete.class.getName());

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.