/**
* Copyright 2010 The Apache Software Foundation
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.alibaba.wasp.fserver;
import com.alibaba.wasp.DeserializationException;
import com.alibaba.wasp.EntityGroupInfo;
import com.alibaba.wasp.EntityGroupTransaction;
import com.alibaba.wasp.FConstants;
import com.alibaba.wasp.MiniWaspCluster;
import com.alibaba.wasp.ServerName;
import com.alibaba.wasp.WaspTestingUtility;
import com.alibaba.wasp.client.WaspAdmin;
import com.alibaba.wasp.executor.EventHandler;
import com.alibaba.wasp.master.EntityGroupPlan;
import com.alibaba.wasp.master.FMaster;
import com.alibaba.wasp.master.handler.SplitEntityGroupHandler;
import com.alibaba.wasp.meta.FMetaReader;
import com.alibaba.wasp.meta.FTable;
import com.alibaba.wasp.meta.TableSchemaCacheReader;
import com.alibaba.wasp.protobuf.ProtobufUtil;
import com.alibaba.wasp.zookeeper.ZKAssign;
import com.alibaba.wasp.zookeeper.ZKUtil;
import com.alibaba.wasp.zookeeper.ZooKeeperWatcher;
import com.google.protobuf.ServiceException;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.zookeeper.KeeperException;
import org.apache.zookeeper.KeeperException.NodeExistsException;
import org.apache.zookeeper.data.Stat;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
import java.io.IOException;
import java.util.List;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
/**
* Like {@link TestSplitTransaction} in that we're testing
* {@link SplitTransaction} only the below tests are against a running cluster
* where {@link TestSplitTransaction} is tests against a bare
* {@link EntityGroup}.
*/
public class TestSplitTransactionOnCluster {
private static final Log LOG = LogFactory
.getLog(TestSplitTransactionOnCluster.class);
private static WaspAdmin admin = null;
private static MiniWaspCluster cluster = null;
private static final int NB_SERVERS = 5;
private static ZooKeeperWatcher zkw;
private static final WaspTestingUtility TESTING_UTIL = new WaspTestingUtility();
@BeforeClass
public static void before() throws Exception {
TESTING_UTIL.getConfiguration().setInt("wasp.balancer.period", 60000);
// Needed because some tests have splits happening on FS that are killed
// We don't want to wait 3min for the master to figure it out
TESTING_UTIL.getConfiguration().setInt(
"wasp.master.assignment.timeoutmonitor.timeout", 4000);
TESTING_UTIL.getConfiguration().setBoolean(
"wasp.online.schema.update.enable", true);
TESTING_UTIL.getConfiguration().set(FConstants.REDO_IMPL,
"com.alibaba.wasp.fserver.redo.MemRedoLog");
TESTING_UTIL.getConfiguration().setInt("wasp.fserver.msginterval", 100);
TESTING_UTIL.getConfiguration().setInt("wasp.client.pause", 250);
TESTING_UTIL.getConfiguration().setInt("wasp.client.retries.number", 6);
TESTING_UTIL.getConfiguration().setBoolean(
"wasp.master.enabletable.roundrobin", true);
TESTING_UTIL.getConfiguration().setBoolean(
"wasp.ipc.server.blacklist.enable", true);
TESTING_UTIL.getConfiguration().setBoolean("wasp.testing.nocluster", false);
TESTING_UTIL.startMiniCluster(NB_SERVERS);
TableSchemaCacheReader.getInstance(TESTING_UTIL.getConfiguration())
.clearCache();
admin = new WaspAdmin(TESTING_UTIL.getConfiguration());
cluster = TESTING_UTIL.getMiniWaspCluster();
zkw = WaspTestingUtility.getZooKeeperWatcher(TESTING_UTIL);
}
@AfterClass
public static void after() throws Exception {
TESTING_UTIL.shutdownMiniCluster();
}
private EntityGroupInfo getAndCheckSingleTableEntityGroup(
final List<EntityGroup> entityGroups) {
assertEquals(1, entityGroups.size());
return entityGroups.get(0).getEntityGroupInfo();
}
/**
* A test that intentionally has master fail the processing of the split
* message. Tests that the fserver split ephemeral node gets cleaned up if it
* crashes and that after we process server shutdown, the daughters are up on
* line.
*
* @throws java.io.IOException
* @throws InterruptedException
* @throws org.apache.zookeeper.KeeperException.NodeExistsException
* @throws org.apache.zookeeper.KeeperException
* @throws com.alibaba.wasp.DeserializationException
*/
@Test
public void testFSSplitEphemeralsDisappearButDaughtersAreOnlinedAfterShutdownHandling()
throws IOException, InterruptedException, NodeExistsException,
KeeperException, DeserializationException, ServiceException {
final byte[] tableName = Bytes.toBytes("ephemeral");
// Create table then get the single entityGroup for our new table.
FTable t = TESTING_UTIL.createTable(tableName);
List<EntityGroup> entityGroups = cluster.getEntityGroups(tableName);
EntityGroupInfo egi = getAndCheckSingleTableEntityGroup(entityGroups);
int tableEntityGroupIndex = getTableEntityGroupIndex(admin, egi);
// Turn off balancer so it doesn't cut in and mess up our placements.
admin.setBalancerRunning(false, true);
try {
// Add a bit of load up into the table so splittable.
TESTING_UTIL.loadTable(t);
// Get entityGroup pre-split.
FServer server = cluster.getFServer(tableEntityGroupIndex);
printOutEntityGroups(server, "Initial entityGroups: ");
int entityGroupCount = ProtobufUtil.getOnlineEntityGroups(server).size();
// Now, before we split, set special flag in master, a flag that has
// it FAIL the processing of split.
SplitEntityGroupHandler.TEST_SKIP = true;
// Now try splitting and it should work.
split(egi, "1", server, entityGroupCount);
// Get daughters
List<EntityGroup> daughters = checkAndGetDaughters(tableName);
// Assert the ephemeral node is up in zk.
String path = ZKAssign.getNodeName(zkw, egi.getEncodedName());
Stat stats = zkw.getRecoverableZooKeeper().exists(path, false);
LOG.info("EPHEMERAL NODE BEFORE SERVER ABORT, path=" + path + ", stats="
+ stats);
EntityGroupTransaction rtd = EntityGroupTransaction.parseFrom(ZKAssign
.getData(zkw, egi.getEncodedName()));
// State could be SPLIT or SPLITTING.
assertTrue(rtd.getEventType().equals(
EventHandler.EventType.FSERVER_ZK_ENTITYGROUP_SPLIT)
|| rtd.getEventType().equals(
EventHandler.EventType.FSERVER_ZK_ENTITYGROUP_SPLITTING));
// Now crash the server
cluster.abortFServer(tableEntityGroupIndex);
waitUntilFServerDead();
awaitDaughters(tableName, daughters.size());
// Assert daughters are online.
entityGroups = cluster.getEntityGroups(tableName);
for (EntityGroup r : entityGroups) {
assertTrue(daughters.contains(r));
}
// Finally assert that the ephemeral SPLIT znode was cleaned up.
for (int i = 0; i < 100; i++) {
// wait a bit (10s max) for the node to disappear
stats = zkw.getRecoverableZooKeeper().exists(path, false);
if (stats == null)
break;
Thread.sleep(100);
}
LOG.info("EPHEMERAL NODE AFTER SERVER ABORT, path=" + path + ", stats="
+ stats);
assertTrue(stats == null);
} finally {
// Set this flag back.
SplitEntityGroupHandler.TEST_SKIP = false;
admin.setBalancerRunning(true, false);
}
}
@Test
public void testExistingZnodeBlocksSplitAndWeRollback() throws IOException,
InterruptedException, NodeExistsException, KeeperException {
final byte[] tableName = Bytes
.toBytes("testExistingZnodeBlocksSplitAndWeRollback");
// Create table then get the single entityGroup for our new table.
FTable t = TESTING_UTIL.createTable(tableName);
// TESTING_UTIL.waitTableEnabled(tableName, 5000);
List<EntityGroup> entityGroups = cluster.getEntityGroups(tableName);
EntityGroupInfo egi = getAndCheckSingleTableEntityGroup(entityGroups);
int tableEntityGroupIndex = getTableEntityGroupIndex(admin, egi);
try {
// Add a bit of load up into the table so splittable.
TESTING_UTIL.loadTable(t);
// Get entityGroup pre-split.
FServer server = cluster.getFServer(tableEntityGroupIndex);
printOutEntityGroups(server, "Initial entityGroups: ");
int entityGroupCount = server.getOnlineEntityGroups().size();
// Insert into zk a blocking znode, a znode of same name as entityGroup
// so it gets in way of our splitting.
ZooKeeperWatcher zkw = WaspTestingUtility
.getZooKeeperWatcher(TESTING_UTIL);
ZKAssign.createNodeClosing(zkw, egi, new ServerName("any.old.server",
1234, -1));
// Now try splitting.... should fail. And each should successfully
// rollback.
admin.split(egi.getEntityGroupNameAsString(), "3");
admin.split(egi.getEntityGroupNameAsString(), "2");
admin.split(egi.getEntityGroupNameAsString(), "1");
// Wait around a while and assert count of entityGroups remains constant.
for (int i = 0; i < 10; i++) {
Thread.sleep(100);
assertEquals(entityGroupCount, server.getOnlineEntityGroups().size());
}
// Now clear the zknode
ZKAssign.deleteClosingNode(zkw, egi);
// Now try splitting and it should work.
split(egi, "0", server, entityGroupCount);
// Get daughters
List<EntityGroup> daughters = cluster.getEntityGroups(tableName);
assertTrue(daughters.size() >= 2);
// OK, so split happened after we cleared the blocking node.
} finally {
//
}
}
/**
* Messy test that simulates case where SplitTransactions fails to add one of
* the daughters up into the .META. table before crash. We're testing fact
* that the shutdown handler will fixup the missing daughter entityGroup
* adding it back into .META.
*
* @throws java.io.IOException
* @throws InterruptedException
*/
@Test
public void testShutdownSimpleFixup() throws IOException,
InterruptedException {
final byte[] tableName = Bytes.toBytes("testShutdownSimpleFixup");
// Create table then get the single entityGroup for our new table.
FTable t = TESTING_UTIL.createTable(tableName);
TESTING_UTIL.waitTableEnabled(tableName, 5000);
List<EntityGroup> entityGroups = cluster.getEntityGroups(tableName);
EntityGroupInfo egi = getAndCheckSingleTableEntityGroup(entityGroups);
int tableEntityGroupIndex = getTableEntityGroupIndex(admin, egi);
try {
// Add a bit of load up into the table so splittable.
TESTING_UTIL.loadTable(t);
// Get entityGroup pre-split.
FServer server = cluster.getFServer(tableEntityGroupIndex);
printOutEntityGroups(server, "Initial entityGroups: ");
int entityGroupCount = server.getOnlineEntityGroups().size();
// Now split.
split(egi, "1", server, entityGroupCount);
// Get daughters
List<EntityGroup> daughters = cluster.getEntityGroups(tableName);
assertTrue(daughters.size() >= 2);
// Remove one of the daughters from .META. to simulate failed insert of
// daughter entityGroup up into .META.
// removeDaughterFromMeta(daughters.get(0).getEntityGroupInfo());
// Now crash the server
cluster.abortFServer(tableEntityGroupIndex);
waitUntilFServerDead();
// Wait till entityGroups are back on line again.
while (cluster.getEntityGroups(tableName).size() < daughters.size()) {
LOG.info("Waiting for repair to happen");
Thread.sleep(1000);
}
// Assert daughters are online.
entityGroups = cluster.getEntityGroups(tableName);
for (EntityGroup r : entityGroups) {
assertTrue(daughters.contains(r));
}
} finally {
//
}
}
/**
* Test that if daughter split on us, we won't do the shutdown handler fixup
* just because we can't find the immediate daughter of an offlined parent.
*
* @throws java.io.IOException
* @throws InterruptedException
*/
@Test
public void testShutdownFixupWhenDaughterHasSplit() throws IOException,
InterruptedException {
final byte[] tableName = Bytes
.toBytes("testShutdownFixupWhenDaughterHasSplit");
// Create table then get the single entityGroup for our new table.
FTable t = TESTING_UTIL.createTable(tableName);
TESTING_UTIL.waitTableEnabled(tableName, 5000);
List<EntityGroup> entityGroups = cluster.getEntityGroups(tableName);
EntityGroupInfo egi = getAndCheckSingleTableEntityGroup(entityGroups);
int tableEntityGroupIndex = getTableEntityGroupIndex(admin, egi);
try {
// Add a bit of load up into the table so splittable.
TESTING_UTIL.loadTable(t);
// Get entityGroup pre-split.
FServer server = cluster.getFServer(tableEntityGroupIndex);
printOutEntityGroups(server, "Initial entityGroups: ");
int entityGroupCount = server.getOnlineEntityGroups().size();
// Now split.
split(egi, "2", server, entityGroupCount);
// Get daughters
List<EntityGroup> daughters = cluster.getEntityGroups(tableName);
assertTrue(daughters.size() >= 2);
// Now split one of the daughters.
entityGroupCount = server.getOnlineEntityGroups().size();
EntityGroupInfo daughter = null;
for (EntityGroup entityGroup : daughters) {
if (!entityGroup.getEntityGroupInfo().isSplit()) {
try {
entityGroup.checkRow(Bytes.toBytes("1"), "find eg");
daughter = entityGroup.getEntityGroupInfo();
break;
} catch (WrongEntityGroupException e) {
}
}
}
assertTrue(daughter != null);
daughters = cluster.getEntityGroups(tableName);
EntityGroup daughterEntityGroup = null;
for (EntityGroup eg : daughters) {
if (eg.getEntityGroupInfo().equals(daughter)) {
daughterEntityGroup = eg;
}
}
assertTrue(daughterEntityGroup != null);
split(daughter, "1", server, entityGroupCount);
// Get list of daughters
daughters = cluster.getEntityGroups(tableName);
// Now crash the server
cluster.abortFServer(tableEntityGroupIndex);
waitUntilFServerDead();
// Wait till entityGroups are back on line again.
while (cluster.getEntityGroups(tableName).size() < daughters.size()) {
LOG.info("Waiting for repair to happen");
Thread.sleep(1000);
}
// Assert daughters are online and ONLY the original daughters -- that
// fixup didn't insert one during server shutdown recover.
entityGroups = cluster.getEntityGroups(tableName);
assertEquals(daughters.size(), entityGroups.size());
for (EntityGroup r : entityGroups) {
assertTrue(daughters.contains(r));
}
} finally {
// ;
}
}
private int getTableEntityGroupIndex(WaspAdmin admin, EntityGroupInfo egi) {
MiniWaspCluster cluster = TESTING_UTIL.getMiniWaspCluster();
int tableEntityGroupIndex = cluster.getServerWith(egi.getEntityGroupName());
assertTrue(tableEntityGroupIndex != -1);
cluster.getFServer(tableEntityGroupIndex);
tableEntityGroupIndex = cluster.getServerWith(egi.getEntityGroupName());
assertTrue(tableEntityGroupIndex != -1);
return tableEntityGroupIndex;
}
/**
* When splitting is partially done and the master goes down when the SPLIT
* node is in either SPLIT or SPLITTING state.
*
* @throws java.io.IOException
* @throws InterruptedException
* @throws org.apache.zookeeper.KeeperException.NodeExistsException
* @throws org.apache.zookeeper.KeeperException
*/
@Test
public void testMasterRestartWhenSplittingIsPartial() throws IOException,
InterruptedException, NodeExistsException, KeeperException,
DeserializationException {
final byte[] tableName = Bytes
.toBytes("testMasterRestartWhenSplittingIsPartial");
// Create table then get the single entityGroup for our new table.
FTable t = TESTING_UTIL.createTable(tableName);
TESTING_UTIL.waitTableEnabled(tableName, 5000);
List<EntityGroup> entityGroups = cluster.getEntityGroups(tableName);
EntityGroupInfo egi = getAndCheckSingleTableEntityGroup(entityGroups);
int tableEntityGroupIndex = getTableEntityGroupIndex(admin, egi);
try {
// Add a bit of load up into the table so splittable.
TESTING_UTIL.loadTable(t);
// Get entityGroup pre-split.
FServer server = cluster.getFServer(tableEntityGroupIndex);
printOutEntityGroups(server, "Initial entityGroups: ");
int entityGroupCount = server.getOnlineEntityGroups().size();
// Now, before we split, set special flag in master, a flag that has
// it FAIL the processing of split.
SplitEntityGroupHandler.TEST_SKIP = true;
// Now try splitting and it should work.
split(egi, "1", server, entityGroupCount);
// Get daughters
List<EntityGroup> daughters = cluster.getEntityGroups(tableName);
assertTrue(daughters.size() >= 2);
// Assert the ephemeral node is up in zk.
String path = ZKAssign.getNodeName(zkw, egi.getEncodedName());
Stat stats = zkw.getRecoverableZooKeeper().exists(path, false);
LOG.info("EPHEMERAL NODE BEFORE SERVER ABORT, path=" + path + ", stats="
+ stats);
EntityGroupTransaction rtd = EntityGroupTransaction.parseFrom(ZKAssign
.getData(zkw, egi.getEncodedName()));
// State could be SPLIT or SPLITTING.
assertTrue(rtd.getEventType().equals(
EventHandler.EventType.FSERVER_ZK_ENTITYGROUP_SPLIT)
|| rtd.getEventType().equals(
EventHandler.EventType.FSERVER_ZK_ENTITYGROUP_SPLITTING));
// abort and wait for new master.
FMaster master = abortAndWaitForMaster();
admin = new WaspAdmin(TESTING_UTIL.getConfiguration());
// update the egi to be offlined and splitted.
egi.setOffline(true);
egi.setSplit(true);
EntityGroupPlan egPlan = master.getAssignmentManager()
.getEntityGroupReopenPlan(egi);
assertTrue(egPlan != null);
} catch (Exception e) {
e.printStackTrace();
} finally {
// Set this flag back.
SplitEntityGroupHandler.TEST_SKIP = false;
}
}
/**
* While transitioning node from FS_ZK_ENTITYGROUP_SPLITTING to
* FS_ZK_ENTITYGROUP_SPLITTING during entityGroup split,if zookeper went down
* split always fails for the entityGroup.This test case is to test the znode
* is deleted(if created) or not in roll back.
*
* @throws java.io.IOException
* @throws InterruptedException
* @throws org.apache.zookeeper.KeeperException
*/
@Test
public void testSplitBeforeSettingSplittingInZK() throws IOException,
InterruptedException, KeeperException {
final byte[] tableName = Bytes
.toBytes("testSplitBeforeSettingSplittingInZK");
TESTING_UTIL.createTable(tableName);
TESTING_UTIL.waitTableEnabled(tableName, 5000);
testSplitBeforeSettingSplittingInZK(tableName, true, false);
testSplitBeforeSettingSplittingInZK(tableName, false, true);
}
private void testSplitBeforeSettingSplittingInZK(byte[] tableName,
boolean nodeCreated, boolean delete) throws IOException, KeeperException,
InterruptedException {
try {
// Create table then get the single entityGroup for our new table.
List<EntityGroup> entityGroups = cluster.getEntityGroups(tableName);
int fserverIndex = cluster.getServerWith(entityGroups.get(0)
.getEntityGroupName());
FServer fserver = cluster.getFServer(fserverIndex);
SplitTransaction st = null;
if (nodeCreated) {
st = new MockedSplitTransaction(entityGroups.get(0), null) {
// @Override
int transitionNodeSplitting(ZooKeeperWatcher zkw,
EntityGroupInfo parent, ServerName serverName, int version)
throws KeeperException, IOException {
throw new IOException();
}
};
} else {
st = new MockedSplitTransaction(entityGroups.get(0), null) {
@Override
int createNodeSplitting(ZooKeeperWatcher zkw,
EntityGroupInfo entityGroup, ServerName serverName)
throws KeeperException, IOException {
throw new IOException();
}
};
}
try {
st.execute(fserver, fserver);
} catch (IOException e) {
String node = ZKAssign.getNodeName(fserver.getZooKeeper(), entityGroups
.get(0).getEntityGroupInfo().getEncodedName());
if (nodeCreated) {
assertFalse(ZKUtil.checkExists(fserver.getZooKeeper(), node) == -1);
} else {
assertTrue(ZKUtil.checkExists(fserver.getZooKeeper(), node) == -1);
}
assertTrue(st.rollback(fserver, fserver));
assertTrue(ZKUtil.checkExists(fserver.getZooKeeper(), node) == -1);
}
} finally {
if (delete) {
if (admin.isTableAvailable(tableName)
&& admin.isTableEnabled(tableName)) {
admin.disableTable(tableName);
admin.deleteTable(tableName);
}
}
}
}
public static class MockedSplitTransaction extends SplitTransaction {
public MockedSplitTransaction(EntityGroup r, byte[] splitrow)
throws IOException {
super(r, splitrow, cluster.getConfiguration());
}
}
private FMaster abortAndWaitForMaster() throws IOException,
InterruptedException {
cluster.abortMaster(0);
cluster.waitOnMaster(0);
cluster.getConfiguration().setClass(FConstants.MASTER_IMPL, FMaster.class,
FMaster.class);
FMaster master = cluster.startMaster().getMaster();
cluster.waitForActiveAndReadyMaster();
return master;
}
private void split(final EntityGroupInfo egi, final String splitPoint,
final FServer server, final int entityGroupCount) throws IOException,
InterruptedException {
admin.split(egi.getEntityGroupNameAsString(), splitPoint);
while (server.getOnlineEntityGroups().size() <= entityGroupCount) {
LOG.debug("Waiting on entityGroup to split");
Thread.sleep(100);
}
}
@Test
public void testTableExistsIfTheSpecifiedTableEntityGroupIsSplitParent()
throws Exception {
final byte[] tableName = Bytes
.toBytes("testTableExistsIfTheSpecifiedTableEntityGroupIsSplitParent");
FServer fServer = null;
List<EntityGroup> entityGroups = null;
WaspAdmin admin = new WaspAdmin(TESTING_UTIL.getConfiguration());
try {
// Create table then get the single entityGroup for our new table.
FTable t = TESTING_UTIL.createTable(tableName);
TESTING_UTIL.waitTableEnabled(tableName, 5000);
entityGroups = cluster.getEntityGroups(tableName);
int fserverIndex = cluster.getServerWith(entityGroups.get(0)
.getEntityGroupName());
fServer = cluster.getFServer(fserverIndex);
insertData(tableName, admin, t);
// Turn off balancer so it doesn't cut in and mess up our placements.
boolean tableExists = FMetaReader.tableExists(
TESTING_UTIL.getConfiguration(), Bytes.toString(tableName));
assertEquals("The specified table should present.", true, tableExists);
SplitTransaction st = new SplitTransaction(entityGroups.get(0),
Bytes.toBytes("row2"), TESTING_UTIL.getConfiguration());
try {
st.prepare();
st.createDaughters(fServer, fServer);
} catch (IOException e) {
}
tableExists = FMetaReader.tableExists(TESTING_UTIL.getConfiguration(),
Bytes.toString(tableName));
assertEquals("The specified table should present.", true, tableExists);
} finally {
admin.close();
}
}
private void insertData(final byte[] tableName, WaspAdmin admin, FTable t)
throws IOException, InterruptedException {
}
private void printOutEntityGroups(final FServer fs, final String prefix)
throws IOException {
for (EntityGroup entityGroup : fs.getOnlineEntityGroups()) {
LOG.info(prefix + entityGroup.getEntityGroupNameAsString());
}
}
private List<EntityGroup> checkAndGetDaughters(byte[] tableName)
throws InterruptedException {
List<EntityGroup> daughters = null;
// try up to 10s
for (int i = 0; i < 100; i++) {
daughters = cluster.getEntityGroups(tableName);
if (daughters.size() >= 2)
break;
Thread.sleep(100);
}
assertTrue(daughters.size() >= 2);
return daughters;
}
private void waitUntilFServerDead() throws InterruptedException {
// Wait until the master processes the FS shutdown
// while (cluster.getMaster().getClusterStatus().getServers().size() ==
// NB_SERVERS) {
// LOG.info("Waiting on server to go down");
// Thread.sleep(100);
// }
for (int i = 0; cluster.getMaster().getClusterStatus().getServers().size() == NB_SERVERS
&& i < 100; i++) {
LOG.info("Waiting on server to go down");
Thread.sleep(100);
}
assertFalse("Waited too long for FS to die", cluster.getMaster()
.getClusterStatus().getServers().size() == NB_SERVERS);
}
private void awaitDaughters(byte[] tableName, int numDaughters)
throws InterruptedException {
// Wait till entityGroups are back on line again.
for (int i = 0; cluster.getEntityGroups(tableName).size() < numDaughters
&& i < 60; i++) {
LOG.info("Waiting for repair to happen");
Thread.sleep(1000);
}
if (cluster.getEntityGroups(tableName).size() < numDaughters) {
fail("Waiting too long for daughter entityGroups");
}
}
}