/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.fs.slive;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import java.io.DataInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Random;
import java.util.Set;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.slive.ArgumentParser.ParsedOutput;
import org.apache.hadoop.fs.slive.Constants.OperationType;
import org.apache.hadoop.fs.slive.DataVerifier.VerifyOutput;
import org.apache.hadoop.fs.slive.DataWriter.GenerateOutput;
import org.apache.hadoop.util.ToolRunner;
import org.junit.Before;
import org.junit.Test;
/**
* Junit 4 test for slive
*/
public class TestSlive {
private static final Log LOG = LogFactory.getLog(TestSlive.class);
private static final Random rnd = new Random(1L);
private static final String TEST_DATA_PROP = "test.build.data";
private static Configuration getBaseConfig() {
Configuration conf = new Configuration();
return conf;
}
/** gets the test write location according to the coding guidelines */
private static File getWriteLoc() {
String writeLoc = System.getProperty(TEST_DATA_PROP, "build/test/data/");
File writeDir = new File(writeLoc, "slive");
writeDir.mkdirs();
return writeDir;
}
/** gets where the MR job places its data + output + results */
private static File getFlowLocation() {
return new File(getWriteLoc(), "flow");
}
/** gets the test directory which is created by the mkdir op */
private static File getTestDir() {
return new File(getWriteLoc(), "slivedir");
}
/**
* gets the test file location
* which is used for reading, appending and created
*/
private static File getTestFile() {
return new File(getWriteLoc(), "slivefile");
}
/**
* gets the rename file which is used in combination
* with the test file to do a rename operation
*/
private static File getTestRenameFile() {
return new File(getWriteLoc(), "slivefile1");
}
/** gets the MR result file name */
private static File getResultFile() {
return new File(getWriteLoc(), "sliveresfile");
}
private static File getImaginaryFile() {
return new File(getWriteLoc(), "slivenofile");
}
/** gets the test program arguments used for merging and main MR running */
private String[] getTestArgs(boolean sleep) {
List<String> args = new LinkedList<String>();
// setup the options
{
args.add("-" + ConfigOption.WRITE_SIZE.getOpt());
args.add("1M,2M");
args.add("-" + ConfigOption.OPS.getOpt());
args.add(Constants.OperationType.values().length + "");
args.add("-" + ConfigOption.MAPS.getOpt());
args.add("2");
args.add("-" + ConfigOption.REDUCES.getOpt());
args.add("2");
args.add("-" + ConfigOption.APPEND_SIZE.getOpt());
args.add("1M,2M");
args.add("-" + ConfigOption.BLOCK_SIZE.getOpt());
args.add("1M,2M");
args.add("-" + ConfigOption.REPLICATION_AM.getOpt());
args.add("1,1");
if (sleep) {
args.add("-" + ConfigOption.SLEEP_TIME.getOpt());
args.add("10,10");
}
args.add("-" + ConfigOption.RESULT_FILE.getOpt());
args.add(getResultFile().toString());
args.add("-" + ConfigOption.BASE_DIR.getOpt());
args.add(getFlowLocation().toString());
args.add("-" + ConfigOption.DURATION.getOpt());
args.add("10");
args.add("-" + ConfigOption.DIR_SIZE.getOpt());
args.add("10");
args.add("-" + ConfigOption.FILES.getOpt());
args.add("10");
}
return args.toArray(new String[args.size()]);
}
@Test
public void testFinder() throws Exception {
ConfigExtractor extractor = getTestConfig(false);
PathFinder fr = new PathFinder(extractor, rnd);
// should only be able to select 10 files
// attempt for a given amount of iterations
int maxIterations = 10000;
Set<Path> files = new HashSet<Path>();
for (int i = 0; i < maxIterations; i++) {
files.add(fr.getFile());
}
assertTrue(files.size() == 10);
Set<Path> dirs = new HashSet<Path>();
for (int i = 0; i < maxIterations; i++) {
dirs.add(fr.getDirectory());
}
assertTrue(dirs.size() == 10);
}
@Test
public void testSelection() throws Exception {
ConfigExtractor extractor = getTestConfig(false);
WeightSelector selector = new WeightSelector(extractor, rnd);
// should be 1 of each type - uniform
int expected = OperationType.values().length;
Operation op = null;
Set<String> types = new HashSet<String>();
FileSystem fs = FileSystem.get(extractor.getConfig());
while (true) {
op = selector.select(1, 1);
if (op == null) {
break;
}
// doesn't matter if they work or not
op.run(fs);
types.add(op.getType());
}
assertEquals(types.size(), expected);
}
// gets the config merged with the arguments
private ConfigExtractor getTestConfig(boolean sleep) throws Exception {
ArgumentParser parser = new ArgumentParser(getTestArgs(sleep));
ParsedOutput out = parser.parse();
assertTrue(!out.shouldOutputHelp());
ConfigMerger merge = new ConfigMerger();
Configuration cfg = merge.getMerged(out, getBaseConfig());
ConfigExtractor extractor = new ConfigExtractor(cfg);
return extractor;
}
@Before
public void ensureDeleted() throws Exception {
rDelete(getTestFile());
rDelete(getTestDir());
rDelete(getTestRenameFile());
rDelete(getResultFile());
rDelete(getFlowLocation());
rDelete(getImaginaryFile());
}
/** cleans up a file or directory recursively if need be */
private void rDelete(File place) throws Exception {
if (place.isFile()) {
LOG.info("Deleting file " + place);
assertTrue(place.delete());
} else if (place.isDirectory()) {
deleteDir(place);
}
}
/** deletes a dir and its contents */
private void deleteDir(File dir) throws Exception {
String fns[] = dir.list();
// delete contents first
for (String afn : fns) {
File fn = new File(dir, afn);
rDelete(fn);
}
LOG.info("Deleting directory " + dir);
// now delete the dir
assertTrue(dir.delete());
}
@Test
public void testArguments() throws Exception {
ConfigExtractor extractor = getTestConfig(true);
assertEquals(extractor.getOpCount().intValue(), Constants.OperationType
.values().length);
assertEquals(extractor.getMapAmount().intValue(), 2);
assertEquals(extractor.getReducerAmount().intValue(), 2);
Range<Long> apRange = extractor.getAppendSize();
assertEquals(apRange.getLower().intValue(), Constants.MEGABYTES * 1);
assertEquals(apRange.getUpper().intValue(), Constants.MEGABYTES * 2);
Range<Long> wRange = extractor.getWriteSize();
assertEquals(wRange.getLower().intValue(), Constants.MEGABYTES * 1);
assertEquals(wRange.getUpper().intValue(), Constants.MEGABYTES * 2);
Range<Long> bRange = extractor.getBlockSize();
assertEquals(bRange.getLower().intValue(), Constants.MEGABYTES * 1);
assertEquals(bRange.getUpper().intValue(), Constants.MEGABYTES * 2);
String resfile = extractor.getResultFile();
assertEquals(resfile, getResultFile().toString());
int durationMs = extractor.getDurationMilliseconds();
assertEquals(durationMs, 10 * 1000);
}
@Test
public void testDataWriting() throws Exception {
long byteAm = 100;
File fn = getTestFile();
DataWriter writer = new DataWriter(rnd);
FileOutputStream fs = new FileOutputStream(fn);
GenerateOutput ostat = writer.writeSegment(byteAm, fs);
LOG.info(ostat);
fs.close();
assertTrue(ostat.getBytesWritten() == byteAm);
DataVerifier vf = new DataVerifier();
FileInputStream fin = new FileInputStream(fn);
VerifyOutput vfout = vf.verifyFile(byteAm, new DataInputStream(fin));
LOG.info(vfout);
fin.close();
assertEquals(vfout.getBytesRead(), byteAm);
assertTrue(vfout.getChunksDifferent() == 0);
}
@Test
public void testRange() {
Range<Long> r = new Range<Long>(10L, 20L);
assertEquals(r.getLower().longValue(), 10L);
assertEquals(r.getUpper().longValue(), 20L);
}
@Test
public void testCreateOp() throws Exception {
// setup a valid config
ConfigExtractor extractor = getTestConfig(false);
final Path fn = new Path(getTestFile().getCanonicalPath());
CreateOp op = new CreateOp(extractor, rnd) {
protected Path getCreateFile() {
return fn;
}
};
runOperationOk(extractor, op, true);
}
@Test
public void testOpFailures() throws Exception {
ConfigExtractor extractor = getTestConfig(false);
final Path fn = new Path(getImaginaryFile().getCanonicalPath());
ReadOp rop = new ReadOp(extractor, rnd) {
protected Path getReadFile() {
return fn;
}
};
runOperationBad(extractor, rop);
DeleteOp dop = new DeleteOp(extractor, rnd) {
protected Path getDeleteFile() {
return fn;
}
};
runOperationBad(extractor, dop);
RenameOp reop = new RenameOp(extractor, rnd) {
protected SrcTarget getRenames() {
return new SrcTarget(fn, fn);
}
};
runOperationBad(extractor, reop);
AppendOp aop = new AppendOp(extractor, rnd) {
protected Path getAppendFile() {
return fn;
}
};
runOperationBad(extractor, aop);
}
private void runOperationBad(ConfigExtractor cfg, Operation op)
throws Exception {
FileSystem fs = FileSystem.get(cfg.getConfig());
List<OperationOutput> data = op.run(fs);
assertTrue(!data.isEmpty());
boolean foundFail = false;
for (OperationOutput d : data) {
if (d.getMeasurementType().equals(ReportWriter.FAILURES)) {
foundFail = true;
}
if (d.getMeasurementType().equals(ReportWriter.NOT_FOUND)) {
foundFail = true;
}
}
assertTrue(foundFail);
}
private void runOperationOk(ConfigExtractor cfg, Operation op, boolean checkOk)
throws Exception {
FileSystem fs = FileSystem.get(cfg.getConfig());
List<OperationOutput> data = op.run(fs);
assertTrue(!data.isEmpty());
if (checkOk) {
boolean foundSuc = false;
boolean foundOpCount = false;
boolean foundTime = false;
for (OperationOutput d : data) {
assertTrue(!d.getMeasurementType().equals(ReportWriter.FAILURES));
if (d.getMeasurementType().equals(ReportWriter.SUCCESSES)) {
foundSuc = true;
}
if (d.getMeasurementType().equals(ReportWriter.OP_COUNT)) {
foundOpCount = true;
}
if (d.getMeasurementType().equals(ReportWriter.OK_TIME_TAKEN)) {
foundTime = true;
}
}
assertTrue(foundSuc);
assertTrue(foundOpCount);
assertTrue(foundTime);
}
}
@Test
public void testDelete() throws Exception {
ConfigExtractor extractor = getTestConfig(false);
final Path fn = new Path(getTestFile().getCanonicalPath());
// ensure file created before delete
CreateOp op = new CreateOp(extractor, rnd) {
protected Path getCreateFile() {
return fn;
}
};
runOperationOk(extractor, op, true);
// now delete
DeleteOp dop = new DeleteOp(extractor, rnd) {
protected Path getDeleteFile() {
return fn;
}
};
runOperationOk(extractor, dop, true);
}
@Test
public void testRename() throws Exception {
ConfigExtractor extractor = getTestConfig(false);
final Path src = new Path(getTestFile().getCanonicalPath());
final Path tgt = new Path(getTestRenameFile().getCanonicalPath());
// ensure file created before rename
CreateOp op = new CreateOp(extractor, rnd) {
protected Path getCreateFile() {
return src;
}
};
runOperationOk(extractor, op, true);
RenameOp rop = new RenameOp(extractor, rnd) {
protected SrcTarget getRenames() {
return new SrcTarget(src, tgt);
}
};
runOperationOk(extractor, rop, true);
}
@Test
public void testMRFlow() throws Exception {
ConfigExtractor extractor = getTestConfig(false);
SliveTest s = new SliveTest(getBaseConfig());
int ec = ToolRunner.run(s, getTestArgs(false));
assertTrue(ec == 0);
String resFile = extractor.getResultFile();
File fn = new File(resFile);
assertTrue(fn.exists());
// can't validate completely since operations may fail (mainly anyone but
// create +mkdir) since they may not find there files
}
@Test
public void testRead() throws Exception {
ConfigExtractor extractor = getTestConfig(false);
final Path fn = new Path(getTestFile().getCanonicalPath());
// ensure file created before read
CreateOp op = new CreateOp(extractor, rnd) {
protected Path getCreateFile() {
return fn;
}
};
runOperationOk(extractor, op, true);
ReadOp rop = new ReadOp(extractor, rnd) {
protected Path getReadFile() {
return fn;
}
};
runOperationOk(extractor, rop, true);
}
@Test
public void testSleep() throws Exception {
ConfigExtractor extractor = getTestConfig(true);
SleepOp op = new SleepOp(extractor, rnd);
runOperationOk(extractor, op, true);
}
@Test
public void testList() throws Exception {
// ensure dir made
ConfigExtractor extractor = getTestConfig(false);
final Path dir = new Path(getTestDir().getCanonicalPath());
MkdirOp op = new MkdirOp(extractor, rnd) {
protected Path getDirectory() {
return dir;
}
};
runOperationOk(extractor, op, true);
// list it
ListOp lop = new ListOp(extractor, rnd) {
protected Path getDirectory() {
return dir;
}
};
runOperationOk(extractor, lop, true);
}
@Test
public void testBadChunks() throws Exception {
File fn = getTestFile();
int byteAm = 10000;
FileOutputStream fout = new FileOutputStream(fn);
byte[] bytes = new byte[byteAm];
rnd.nextBytes(bytes);
fout.write(bytes);
fout.close();
// attempt to read it
DataVerifier vf = new DataVerifier();
VerifyOutput vout = new VerifyOutput(0, 0, 0, 0);
DataInputStream in = null;
try {
in = new DataInputStream(new FileInputStream(fn));
vout = vf.verifyFile(byteAm, in);
} catch (Exception e) {
} finally {
if(in != null) in.close();
}
assertTrue(vout.getChunksSame() == 0);
}
@Test
public void testMkdir() throws Exception {
ConfigExtractor extractor = getTestConfig(false);
final Path dir = new Path(getTestDir().getCanonicalPath());
MkdirOp op = new MkdirOp(extractor, rnd) {
protected Path getDirectory() {
return dir;
}
};
runOperationOk(extractor, op, true);
}
@Test
public void testSelector() throws Exception {
ConfigExtractor extractor = getTestConfig(false);
RouletteSelector selector = new RouletteSelector(rnd);
List<OperationWeight> sList = new LinkedList<OperationWeight>();
Operation op = selector.select(sList);
assertTrue(op == null);
CreateOp cop = new CreateOp(extractor, rnd);
sList.add(new OperationWeight(cop, 1.0d));
AppendOp aop = new AppendOp(extractor, rnd);
sList.add(new OperationWeight(aop, 0.01d));
op = selector.select(sList);
assertTrue(op == cop);
}
@Test
public void testAppendOp() throws Exception {
// setup a valid config
ConfigExtractor extractor = getTestConfig(false);
// ensure file created before append
final Path fn = new Path(getTestFile().getCanonicalPath());
CreateOp op = new CreateOp(extractor, rnd) {
protected Path getCreateFile() {
return fn;
}
};
runOperationOk(extractor, op, true);
// local file system (ChecksumFileSystem) currently doesn't support append -
// but we'll leave this test here anyways but can't check the results..
AppendOp aop = new AppendOp(extractor, rnd) {
protected Path getAppendFile() {
return fn;
}
};
runOperationOk(extractor, aop, false);
}
}