Examples of org.apache.flume.channel.file.FileChannel

org.apache.flume.channel.file.FileChannel

A durable {@link Channel} implementation that uses the local file system forits storage.

FileChannel works by writing all transactions to a set of directories specified in the configuration. Additionally, when a commit occurs the transaction is synced to disk.

FileChannel is marked {@link org.apache.flume.annotations.InterfaceAudience.Private} because itshould only be instantiated via a configuration. For example, users should certainly use FileChannel but not by instantiating FileChannel objects. Meaning the label Private applies to user-developers not user-operators. In cases where a Channel is required by instantiated by user-developers {@link org.apache.flume.channel.MemoryChannel} should be used.

      reader.close();


    }
    FileChannelIntegrityTool tool = new FileChannelIntegrityTool();
    tool.run(new String[] {"-l", dataDir.toString()});
    FileChannel channel = new FileChannel();
    channel.setName("channel");
    String cp;
    if(withCheckpoint) {
      cp = origCheckpointDir.toString();
    } else {
      FileUtils.deleteDirectory(checkpointDir);
      Assert.assertTrue(checkpointDir.mkdirs());
      cp = checkpointDir.toString();
    }
    ctx.put(FileChannelConfiguration.CHECKPOINT_DIR,cp);
    ctx.put(FileChannelConfiguration.DATA_DIRS, dataDir.toString());
    channel.configure(ctx);
    channel.start();
    Transaction tx = channel.getTransaction();
    tx.begin();
    int i = 0;
    while(channel.take() != null) {
      i++;
    }
    tx.commit();
    tx.close();
    channel.stop();
    Assert.assertEquals(25 - corrupted, i);
    files = dataDir.listFiles(new FilenameFilter() {
      @Override
      public boolean accept(File dir, String name) {
        if(name.contains(".bak")) {

View Full Code Here

    baseDir = Files.createTempDir();
    origCheckpointDir = new File(baseDir, "chkpt");
    Assert.assertTrue(origCheckpointDir.mkdirs() || origCheckpointDir.isDirectory());
    origDataDir = new File(baseDir, "data");
    Assert.assertTrue(origDataDir.mkdirs() || origDataDir.isDirectory());
    FileChannel channel = new FileChannel();
    channel.setName("channel");
    ctx = new Context();
    ctx.put(FileChannelConfiguration.CAPACITY, "1000");
    ctx.put(FileChannelConfiguration.CHECKPOINT_DIR, origCheckpointDir.toString());
    ctx.put(FileChannelConfiguration.DATA_DIRS, origDataDir.toString());
    ctx.put(FileChannelConfiguration.MAX_FILE_SIZE, "10000");
    ctx.put(FileChannelConfiguration.TRANSACTION_CAPACITY, "100");
    channel.configure(ctx);
    channel.start();
    for (int j = 0; j < 5; j++) {
      Transaction tx = channel.getTransaction();
      tx.begin();
      for (int i = 0; i < 5; i++) {
        channel.put(event);
      }
      tx.commit();
      tx.close();
    }
    Log log = field("log")
      .ofType(Log.class)
      .in(channel)
      .get();


    Assert.assertTrue("writeCheckpoint returned false",
      method("writeCheckpoint")
        .withReturnType(Boolean.class)
        .withParameterTypes(Boolean.class)
        .in(log)
        .invoke(true));
    channel.stop();
  }

View Full Code Here

      reader.close();


    }
    FileChannelIntegrityTool tool = new FileChannelIntegrityTool();
    tool.run(new String[] {"-l", dataDir.toString()});
    FileChannel channel = new FileChannel();
    channel.setName("channel");
    String cp;
    if(withCheckpoint) {
      cp = origCheckpointDir.toString();
    } else {
      FileUtils.deleteDirectory(checkpointDir);
      Assert.assertTrue(checkpointDir.mkdirs());
      cp = checkpointDir.toString();
    }
    ctx.put(FileChannelConfiguration.CHECKPOINT_DIR,cp);
    ctx.put(FileChannelConfiguration.DATA_DIRS, dataDir.toString());
    channel.configure(ctx);
    channel.start();
    Transaction tx = channel.getTransaction();
    tx.begin();
    int i = 0;
    while(channel.take() != null) {
      i++;
    }
    tx.commit();
    tx.close();
    channel.stop();
    Assert.assertEquals(25 - corrupted, i);
    files = dataDir.listFiles(new FilenameFilter() {
      @Override
      public boolean accept(File dir, String name) {
        if(name.contains(".bak")) {

View Full Code Here

    baseDir = Files.createTempDir();
    origCheckpointDir = new File(baseDir, "chkpt");
    Assert.assertTrue(origCheckpointDir.mkdirs() || origCheckpointDir.isDirectory());
    origDataDir = new File(baseDir, "data");
    Assert.assertTrue(origDataDir.mkdirs() || origDataDir.isDirectory());
    FileChannel channel = new FileChannel();
    channel.setName("channel");
    ctx = new Context();
    ctx.put(FileChannelConfiguration.CAPACITY, "1000");
    ctx.put(FileChannelConfiguration.CHECKPOINT_DIR, origCheckpointDir.toString());
    ctx.put(FileChannelConfiguration.DATA_DIRS, origDataDir.toString());
    ctx.put(FileChannelConfiguration.MAX_FILE_SIZE, "10000");
    ctx.put(FileChannelConfiguration.TRANSACTION_CAPACITY, "100");
    channel.configure(ctx);
    channel.start();
    for (int j = 0; j < 5; j++) {
      Transaction tx = channel.getTransaction();
      tx.begin();
      for (int i = 0; i < 5; i++) {
        channel.put(event);
      }
      tx.commit();
      tx.close();
    }
    Log log = field("log")
      .ofType(Log.class)
      .in(channel)
      .get();


    Assert.assertTrue("writeCheckpoint returned false",
      method("writeCheckpoint")
        .withReturnType(Boolean.class)
        .withParameterTypes(Boolean.class)
        .in(log)
        .invoke(true));
    channel.stop();
  }

View Full Code Here

TOP

Related Classes of org.apache.flume.channel.file.FileChannel

org.apache.flume.channel.file.Log.Builder

org.apache.flume.Context

org.apache.flume.Event

org.apache.flume.instrumentation.ChannelCounter

org.apache.flume.tools.TestFileChannelIntegrityTool

java.io.FileOutputStream

java.util.concurrent.Semaphore

java.io.File

org.apache.flume.ChannelException

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.