/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hbase.backup;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.PathFilter;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.MediumTests;
import org.apache.hadoop.hbase.Stoppable;
import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.hadoop.hbase.master.cleaner.HFileCleaner;
import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
import org.apache.hadoop.hbase.regionserver.HRegionServer;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.FSUtils;
import org.apache.hadoop.hbase.util.HFileArchiveTestingUtil;
import org.apache.hadoop.hbase.util.HFileArchiveUtil;
import org.apache.hadoop.hbase.util.StoppableImplementation;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.experimental.categories.Category;
/**
* Test that the {@link HFileArchiver} correctly removes all the parts of a region when cleaning up
* a region
*/
@Category(MediumTests.class)
public class TestHFileArchiving {
private static final String STRING_TABLE_NAME = "test_table";
private static final Log LOG = LogFactory.getLog(TestHFileArchiving.class);
private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
private static final byte[] TABLE_NAME = Bytes.toBytes(STRING_TABLE_NAME);
private static final byte[] TEST_FAM = Bytes.toBytes("fam");
/**
* Setup the config for the cluster
*/
@BeforeClass
public static void setupCluster() throws Exception {
setupConf(UTIL.getConfiguration());
UTIL.startMiniCluster();
// We don't want the cleaner to remove files. The tests do that.
UTIL.getMiniHBaseCluster().getMaster().getHFileCleaner().interrupt();
}
private static void setupConf(Configuration conf) {
// disable the ui
conf.setInt("hbase.regionsever.info.port", -1);
// drop the memstore size so we get flushes
conf.setInt("hbase.hregion.memstore.flush.size", 25000);
// disable major compactions
conf.setInt(HConstants.MAJOR_COMPACTION_PERIOD, 0);
// prevent aggressive region split
conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
ConstantSizeRegionSplitPolicy.class.getName());
}
@Before
public void setup() throws Exception {
UTIL.createTable(TABLE_NAME, TEST_FAM);
}
@After
public void tearDown() throws Exception {
// cleanup the cluster if its up still
if (UTIL.getHBaseAdmin().tableExists(STRING_TABLE_NAME)) {
UTIL.deleteTable(TABLE_NAME);
}
// and cleanup the archive directory
try {
clearArchiveDirectory();
} catch (IOException e) {
Assert.fail("Failure to delete archive directory:" + e.getMessage());
}
}
@AfterClass
public static void cleanupTest() throws Exception {
try {
UTIL.shutdownMiniCluster();
} catch (Exception e) {
// NOOP;
}
}
@Test
public void testRemovesRegionDirOnArchive() throws Exception {
final HBaseAdmin admin = UTIL.getHBaseAdmin();
// get the current store files for the region
List<HRegion> servingRegions = UTIL.getHBaseCluster().getRegions(TABLE_NAME);
// make sure we only have 1 region serving this table
assertEquals(1, servingRegions.size());
HRegion region = servingRegions.get(0);
// and load the table
UTIL.loadRegion(region, TEST_FAM);
// shutdown the table so we can manipulate the files
admin.disableTable(STRING_TABLE_NAME);
FileSystem fs = UTIL.getTestFileSystem();
// now attempt to depose the region
Path rootDir = region.getRegionFileSystem().getTableDir().getParent();
Path regionDir = HRegion.getRegionDir(rootDir, region.getRegionInfo());
HFileArchiver.archiveRegion(UTIL.getConfiguration(), fs, region.getRegionInfo());
// check for the existence of the archive directory and some files in it
Path archiveDir = HFileArchiveTestingUtil.getRegionArchiveDir(UTIL.getConfiguration(), region);
assertTrue(fs.exists(archiveDir));
// check to make sure the store directory was copied
FileStatus[] stores = fs.listStatus(archiveDir);
assertTrue(stores.length == 1);
// make sure we archived the store files
FileStatus[] storeFiles = fs.listStatus(stores[0].getPath());
assertTrue(storeFiles.length > 0);
// then ensure the region's directory isn't present
assertFalse(fs.exists(regionDir));
}
/**
* Test that the region directory is removed when we archive a region without store files, but
* still has hidden files.
* @throws Exception
*/
@Test
public void testDeleteRegionWithNoStoreFiles() throws Exception {
// get the current store files for the region
List<HRegion> servingRegions = UTIL.getHBaseCluster().getRegions(TABLE_NAME);
// make sure we only have 1 region serving this table
assertEquals(1, servingRegions.size());
HRegion region = servingRegions.get(0);
FileSystem fs = region.getRegionFileSystem().getFileSystem();
// make sure there are some files in the regiondir
Path rootDir = FSUtils.getRootDir(fs.getConf());
Path regionDir = HRegion.getRegionDir(rootDir, region.getRegionInfo());
FileStatus[] regionFiles = FSUtils.listStatus(fs, regionDir, null);
Assert.assertNotNull("No files in the region directory", regionFiles);
if (LOG.isDebugEnabled()) {
List<Path> files = new ArrayList<Path>();
for (FileStatus file : regionFiles) {
files.add(file.getPath());
}
LOG.debug("Current files:" + files);
}
// delete the visible folders so we just have hidden files/folders
final PathFilter dirFilter = new FSUtils.DirFilter(fs);
PathFilter nonHidden = new PathFilter() {
@Override
public boolean accept(Path file) {
return dirFilter.accept(file) && !file.getName().toString().startsWith(".");
}
};
FileStatus[] storeDirs = FSUtils.listStatus(fs, regionDir, nonHidden);
for (FileStatus store : storeDirs) {
LOG.debug("Deleting store for test");
fs.delete(store.getPath(), true);
}
// then archive the region
HFileArchiver.archiveRegion(UTIL.getConfiguration(), fs, region.getRegionInfo());
// and check to make sure the region directoy got deleted
assertFalse("Region directory (" + regionDir + "), still exists.", fs.exists(regionDir));
}
@Test
public void testArchiveOnTableDelete() throws Exception {
List<HRegion> servingRegions = UTIL.getHBaseCluster().getRegions(TableName.valueOf(TABLE_NAME));
// make sure we only have 1 region serving this table
assertEquals(1, servingRegions.size());
HRegion region = servingRegions.get(0);
// get the parent RS and monitor
HRegionServer hrs = UTIL.getRSForFirstRegionInTable(TABLE_NAME);
FileSystem fs = hrs.getFileSystem();
// put some data on the region
LOG.debug("-------Loading table");
UTIL.loadRegion(region, TEST_FAM);
// get the hfiles in the region
List<HRegion> regions = hrs.getOnlineRegions(TableName.valueOf(TABLE_NAME));
assertEquals("More that 1 region for test table.", 1, regions.size());
region = regions.get(0);
// wait for all the compactions to complete
region.waitForFlushesAndCompactions();
// disable table to prevent new updates
UTIL.getHBaseAdmin().disableTable(TABLE_NAME);
LOG.debug("Disabled table");
// remove all the files from the archive to get a fair comparison
clearArchiveDirectory();
// then get the current store files
List<String> storeFiles = getRegionStoreFiles(region);
// then delete the table so the hfiles get archived
UTIL.deleteTable(TABLE_NAME);
LOG.debug("Deleted table");
assertArchiveFiles(fs, storeFiles, 30000);
}
private void assertArchiveFiles(FileSystem fs, List<String> storeFiles, long timeout) throws IOException {
long end = System.currentTimeMillis() + timeout;
Path archiveDir = HFileArchiveUtil.getArchivePath(UTIL.getConfiguration());
List<String> archivedFiles = new ArrayList<String>();
// We have to ensure that the DeleteTableHandler is finished. HBaseAdmin.deleteXXX() can return before all files
// are archived. We should fix HBASE-5487 and fix synchronous operations from admin.
while (System.currentTimeMillis() < end) {
archivedFiles = getAllFileNames(fs, archiveDir);
if (archivedFiles.size() >= storeFiles.size()) {
break;
}
}
Collections.sort(storeFiles);
Collections.sort(archivedFiles);
LOG.debug("Store files:");
for (int i = 0; i < storeFiles.size(); i++) {
LOG.debug(i + " - " + storeFiles.get(i));
}
LOG.debug("Archive files:");
for (int i = 0; i < archivedFiles.size(); i++) {
LOG.debug(i + " - " + archivedFiles.get(i));
}
assertTrue("Archived files are missing some of the store files!",
archivedFiles.containsAll(storeFiles));
}
/**
* Test that the store files are archived when a column family is removed.
* @throws Exception
*/
@Test
public void testArchiveOnTableFamilyDelete() throws Exception {
List<HRegion> servingRegions = UTIL.getHBaseCluster().getRegions(TABLE_NAME);
// make sure we only have 1 region serving this table
assertEquals(1, servingRegions.size());
HRegion region = servingRegions.get(0);
// get the parent RS and monitor
HRegionServer hrs = UTIL.getRSForFirstRegionInTable(TABLE_NAME);
FileSystem fs = hrs.getFileSystem();
// put some data on the region
LOG.debug("-------Loading table");
UTIL.loadRegion(region, TEST_FAM);
// get the hfiles in the region
List<HRegion> regions = hrs.getOnlineRegions(TableName.valueOf(TABLE_NAME));
assertEquals("More that 1 region for test table.", 1, regions.size());
region = regions.get(0);
// wait for all the compactions to complete
region.waitForFlushesAndCompactions();
// disable table to prevent new updates
UTIL.getHBaseAdmin().disableTable(TABLE_NAME);
LOG.debug("Disabled table");
// remove all the files from the archive to get a fair comparison
clearArchiveDirectory();
// then get the current store files
List<String> storeFiles = getRegionStoreFiles(region);
// then delete the table so the hfiles get archived
UTIL.getHBaseAdmin().deleteColumn(TableName.valueOf(TABLE_NAME), TEST_FAM);
assertArchiveFiles(fs, storeFiles, 30000);
UTIL.deleteTable(TABLE_NAME);
}
/**
* Test HFileArchiver.resolveAndArchive() race condition HBASE-7643
*/
@Test
public void testCleaningRace() throws Exception {
final long TEST_TIME = 20 * 1000;
Configuration conf = UTIL.getMiniHBaseCluster().getMaster().getConfiguration();
Path rootDir = UTIL.getDataTestDir("testCleaningRace");
FileSystem fs = UTIL.getTestFileSystem();
Path archiveDir = new Path(rootDir, HConstants.HFILE_ARCHIVE_DIRECTORY);
Path regionDir = new Path(FSUtils.getTableDir(new Path("./"),
TableName.valueOf("table")), "abcdef");
Path familyDir = new Path(regionDir, "cf");
Path sourceRegionDir = new Path(rootDir, regionDir);
fs.mkdirs(sourceRegionDir);
Stoppable stoppable = new StoppableImplementation();
// The cleaner should be looping without long pauses to reproduce the race condition.
HFileCleaner cleaner = new HFileCleaner(1, stoppable, conf, fs, archiveDir);
try {
cleaner.start();
// Keep creating/archiving new files while the cleaner is running in the other thread
long startTime = System.currentTimeMillis();
for (long fid = 0; (System.currentTimeMillis() - startTime) < TEST_TIME; ++fid) {
Path file = new Path(familyDir, String.valueOf(fid));
Path sourceFile = new Path(rootDir, file);
Path archiveFile = new Path(archiveDir, file);
fs.createNewFile(sourceFile);
try {
// Try to archive the file
HFileArchiver.archiveRegion(fs, rootDir,
sourceRegionDir.getParent(), sourceRegionDir);
// The archiver succeded, the file is no longer in the original location
// but it's in the archive location.
LOG.debug("hfile=" + fid + " should be in the archive");
assertTrue(fs.exists(archiveFile));
assertFalse(fs.exists(sourceFile));
} catch (IOException e) {
// The archiver is unable to archive the file. Probably HBASE-7643 race condition.
// in this case, the file should not be archived, and we should have the file
// in the original location.
LOG.debug("hfile=" + fid + " should be in the source location");
assertFalse(fs.exists(archiveFile));
assertTrue(fs.exists(sourceFile));
// Avoid to have this file in the next run
fs.delete(sourceFile, false);
}
}
} finally {
stoppable.stop("test end");
cleaner.join();
fs.delete(rootDir, true);
}
}
private void clearArchiveDirectory() throws IOException {
UTIL.getTestFileSystem().delete(
new Path(UTIL.getDefaultRootDirPath(), HConstants.HFILE_ARCHIVE_DIRECTORY), true);
}
/**
* Get the names of all the files below the given directory
* @param fs
* @param archiveDir
* @return
* @throws IOException
*/
private List<String> getAllFileNames(final FileSystem fs, Path archiveDir) throws IOException {
FileStatus[] files = FSUtils.listStatus(fs, archiveDir, null);
return recurseOnFiles(fs, files, new ArrayList<String>());
}
/** Recursively lookup all the file names under the file[] array **/
private List<String> recurseOnFiles(FileSystem fs, FileStatus[] files, List<String> fileNames)
throws IOException {
if (files == null || files.length == 0) return fileNames;
for (FileStatus file : files) {
if (file.isDir()) {
recurseOnFiles(fs, FSUtils.listStatus(fs, file.getPath(), null), fileNames);
} else fileNames.add(file.getPath().getName());
}
return fileNames;
}
private List<String> getRegionStoreFiles(final HRegion region) throws IOException {
Path regionDir = region.getRegionFileSystem().getRegionDir();
FileSystem fs = region.getRegionFileSystem().getFileSystem();
List<String> storeFiles = getAllFileNames(fs, regionDir);
// remove all the non-storefile named files for the region
for (int i = 0; i < storeFiles.size(); i++) {
String file = storeFiles.get(i);
if (file.contains(HRegionFileSystem.REGION_INFO_FILE) || file.contains("hlog")) {
storeFiles.remove(i--);
}
}
storeFiles.remove(HRegionFileSystem.REGION_INFO_FILE);
return storeFiles;
}
}