Examples of MemoryChannel

org.apache.flume.channel.MemoryChannel

MemoryChannel is the recommended channel to use when speeds which writing to disk is impractical is required or durability of data is not required.

Additionally, MemoryChannel should be used when a channel is required for unit testing purposes.

Examples of org.apache.flume.channel.MemoryChannel


  @Before
  public void setUp() throws UnknownHostException {
    localhost = InetAddress.getByName("127.0.0.1");
    source = new AvroSource();
    channel = new MemoryChannel();


    Configurables.configure(channel, new Context());


    List<Channel> channels = new ArrayList<Channel>();
    channels.add(channel);

View Full Code Here

Examples of org.apache.flume.channel.MemoryChannel

  public void setup() throws EventDeliveryException {
    Datasets.delete(FILE_DATASET_URI);
    Datasets.create(FILE_DATASET_URI, DESCRIPTOR);


    this.config = new Context();
    this.in = new MemoryChannel();
    Configurables.configure(in, config);


    config.put(DatasetSinkConstants.CONFIG_KITE_DATASET_URI, FILE_DATASET_URI);


    GenericRecordBuilder builder = new GenericRecordBuilder(RECORD_SCHEMA);

View Full Code Here

Examples of org.apache.flume.channel.MemoryChannel

  private final String bodyWithTandH = "<10>" + stamp1 + " " + host1 + " " +
      data1 + "\n";


  private void init(String keepFields){
    source = new SyslogTcpSource();
    channel = new MemoryChannel();


    Configurables.configure(channel, new Context());


    List<Channel> channels = new ArrayList<Channel>();
    channels.add(channel);

View Full Code Here

Examples of org.apache.flume.channel.MemoryChannel

        Boolean.toString(useRawLocalFileSystem));
    context.put("hdfs.fileType", "DataStream");


    Configurables.configure(sink, context);


    Channel channel = new MemoryChannel();
    Configurables.configure(channel, context);


    sink.setChannel(channel);
    sink.start();


    Calendar eventDate = Calendar.getInstance();
    List<String> bodies = Lists.newArrayList();


    // push the event batches into channel to roll twice
    for (i = 1; i <= (rollCount*10)/batchSize; i++) {
      Transaction txn = channel.getTransaction();
      txn.begin();
      for (j = 1; j <= batchSize; j++) {
        Event event = new SimpleEvent();
        eventDate.clear();
        eventDate.set(2011, i, i, i, 0); // yy mm dd
        String body = "Test." + i + "." + j;
        event.setBody(body.getBytes());
        bodies.add(body);
        channel.put(event);
        totalEvents++;
      }
      txn.commit();
      txn.close();

View Full Code Here

Examples of org.apache.flume.channel.MemoryChannel

     * context.get("hdfs.rollSize", String.class); context.get("hdfs.rollCount",
     * String.class);
     */
    Configurables.configure(sink, context);


    sink.setChannel(new MemoryChannel());


    sink.start();
    sink.stop();
  }

View Full Code Here

Examples of org.apache.flume.channel.MemoryChannel

  @Test
  public void testEmptyChannelResultsInStatusBackoff()
      throws InterruptedException, LifecycleException, EventDeliveryException {
    LOG.debug("Starting...");
    Context context = new Context();
    Channel channel = new MemoryChannel();
    context.put("hdfs.path", testPath);
    context.put("keep-alive", "0");
    Configurables.configure(sink, context);
    Configurables.configure(channel, context);
    sink.setChannel(channel);

View Full Code Here

Examples of org.apache.flume.channel.MemoryChannel

    context.put("hdfs.writeFormat", "Text");
    context.put("hdfs.fileType", "DataStream");


    Configurables.configure(sink, context);


    Channel channel = new MemoryChannel();
    Configurables.configure(channel, context);


    sink.setChannel(channel);
    sink.start();


    Calendar eventDate = Calendar.getInstance();
    List<String> bodies = Lists.newArrayList();


    // push the event batches into channel
    for (i = 1; i < 4; i++) {
      Transaction txn = channel.getTransaction();
      txn.begin();
      for (j = 1; j <= batchSize; j++) {
        Event event = new SimpleEvent();
        eventDate.clear();
        eventDate.set(2011, i, i, i, 0); // yy mm dd
        event.getHeaders().put("timestamp",
            String.valueOf(eventDate.getTimeInMillis()));
        event.getHeaders().put("hostname", "Host" + i);
        String body = "Test." + i + "." + j;
        event.setBody(body.getBytes());
        bodies.add(body);
        channel.put(event);
        totalEvents++;
      }
      txn.commit();
      txn.close();

View Full Code Here

Examples of org.apache.flume.channel.MemoryChannel

    context.put("hdfs.fileType", "DataStream");
    context.put("serializer", "AVRO_EVENT");


    Configurables.configure(sink, context);


    Channel channel = new MemoryChannel();
    Configurables.configure(channel, context);


    sink.setChannel(channel);
    sink.start();


    Calendar eventDate = Calendar.getInstance();
    List<String> bodies = Lists.newArrayList();


    // push the event batches into channel
    for (i = 1; i < 4; i++) {
      Transaction txn = channel.getTransaction();
      txn.begin();
      for (j = 1; j <= batchSize; j++) {
        Event event = new SimpleEvent();
        eventDate.clear();
        eventDate.set(2011, i, i, i, 0); // yy mm dd
        event.getHeaders().put("timestamp",
            String.valueOf(eventDate.getTimeInMillis()));
        event.getHeaders().put("hostname", "Host" + i);
        String body = "Test." + i + "." + j;
        event.setBody(body.getBytes());
        bodies.add(body);
        channel.put(event);
        totalEvents++;
      }
      txn.commit();
      txn.close();

View Full Code Here

Examples of org.apache.flume.channel.MemoryChannel

    context.put("hdfs.rollCount", String.valueOf(rollCount));
    context.put("hdfs.batchSize", String.valueOf(batchSize));


    Configurables.configure(sink, context);


    Channel channel = new MemoryChannel();
    Configurables.configure(channel, context);


    sink.setChannel(channel);
    sink.start();


    Calendar eventDate = Calendar.getInstance();
    List<String> bodies = Lists.newArrayList();


    // push the event batches into channel
    for (i = 1; i < numBatches; i++) {
      Transaction txn = channel.getTransaction();
      txn.begin();
      for (j = 1; j <= batchSize; j++) {
        Event event = new SimpleEvent();
        eventDate.clear();
        eventDate.set(2011, i, i, i, 0); // yy mm dd
        event.getHeaders().put("timestamp",
            String.valueOf(eventDate.getTimeInMillis()));
        event.getHeaders().put("hostname", "Host" + i);
        String body = "Test." + i + "." + j;
        event.setBody(body.getBytes());
        bodies.add(body);
        channel.put(event);
        totalEvents++;
      }
      txn.commit();
      txn.close();

View Full Code Here

Examples of org.apache.flume.channel.MemoryChannel

    context.put("hdfs.batchSize", String.valueOf(batchSize));
    context.put("hdfs.useLocalTimeStamp", String.valueOf(true));


    Configurables.configure(sink, context);


    Channel channel = new MemoryChannel();
    Configurables.configure(channel, context);


    sink.setChannel(channel);
    sink.setBucketClock(clk);
    sink.start();


    Calendar eventDate = Calendar.getInstance();
    List<String> bodies = Lists.newArrayList();


    // push the event batches into channel
    for (i = 1; i < numBatches; i++) {
      Transaction txn = channel.getTransaction();
      txn.begin();
      for (j = 1; j <= batchSize; j++) {
        Event event = new SimpleEvent();
        eventDate.clear();
        eventDate.set(2011, i, i, i, 0); // yy mm dd
        event.getHeaders().put("timestamp",
          String.valueOf(eventDate.getTimeInMillis()));
        event.getHeaders().put("hostname", "Host" + i);
        String body = "Test." + i + "." + j;
        event.setBody(body.getBytes());
        bodies.add(body);
        channel.put(event);
        totalEvents++;
      }
      txn.commit();
      txn.close();

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.