String nnURL = getNameNodeURL(cluster);
logger.info("Namenode address: {}", nnURL);
Context chanCtx = new Context();
MemoryChannel channel = new MemoryChannel();
channel.setName("simpleHDFSTest-mem-chan");
channel.configure(chanCtx);
channel.start();
Context sinkCtx = new Context();
sinkCtx.put("hdfs.path", nnURL + outputDir);
sinkCtx.put("hdfs.fileType", HDFSWriterFactory.DataStreamType);
sinkCtx.put("hdfs.batchSize", Integer.toString(1));
HDFSEventSink sink = new HDFSEventSink();
sink.setName("simpleHDFSTest-hdfs-sink");
sink.configure(sinkCtx);
sink.setChannel(channel);
sink.start();
// create an event
channel.getTransaction().begin();
try {
channel.put(EventBuilder.withBody("yarg 1", Charsets.UTF_8));
channel.put(EventBuilder.withBody("yarg 2", Charsets.UTF_8));
channel.put(EventBuilder.withBody("yarg 3", Charsets.UTF_8));
channel.put(EventBuilder.withBody("yarg 4", Charsets.UTF_8));
channel.put(EventBuilder.withBody("yarg 5", Charsets.UTF_8));
channel.put(EventBuilder.withBody("yarg 5", Charsets.UTF_8));
channel.getTransaction().commit();
} finally {
channel.getTransaction().close();
}
// store events to HDFS
logger.info("Running process(). Create new file.");
sink.process(); // create new file;
logger.info("Running process(). Same file.");
sink.process();
// kill a datanode
logger.info("Killing datanode #1...");
cluster.stopDataNode(0);
// there is a race here.. the client may or may not notice that the
// datanode is dead before it next sync()s.
// so, this next call may or may not roll a new file.
logger.info("Running process(). Create new file? (racy)");
sink.process();
logger.info("Running process(). Create new file.");
sink.process();
logger.info("Running process(). Create new file.");
sink.process();
logger.info("Running process(). Create new file.");
sink.process();
// shut down flume
sink.stop();
channel.stop();
// verify that it's in HDFS and that its content is what we say it should be
FileStatus[] statuses = fs.listStatus(outputDirPath);
Assert.assertNotNull("No files found written to HDFS", statuses);