Examples of com.cloudera.flume.conf.Context

com.cloudera.flume.conf.Context
This context is an abstraction for scopes. Prior to this there were one scopes the physical node (essentially conf file). This allows for new subscopes such as logical node contexts. This allows information to be passed to specific logical nodes (such as logicalnode name). This interface will likely subsume FlumeConfiguration (physical node scope), and configuration information grabbed from the master such as as logical node specific failover chains (master scope). With scope chaining and shadowing we can in the future introduce node specific configuration settings transparently.

        try {
          List<FlumeNodeSpec> cfgs = FlumeSpecGen.generate(argv[0]);
          // check all cfgs to make sure they are valid
          for (FlumeNodeSpec spec : cfgs) {
            // TODO: first arg should be physical node
            Context ctx = new LogicalNodeContext(spec.node, spec.node);
            FlumeBuilder.buildSource(ctx, spec.src);
            FlumeBuilder.buildSink(ctx, spec.sink);
          }


          // set all cfgs to make sure they are valid.

View Full Code Here

      .getLogger(TestTextFileSink.class);


  @Test
  public void testTextKWArg() throws RecognitionException, FlumeSpecException {
    String s = "text(\"filename\", format=\"raw\")";
    TextFileSink snk = (TextFileSink) FlumeBuilder.buildSink(new Context(), s);
    assertTrue(snk.getFormat() instanceof RawOutputFormat);


    s = "text(\"filename\", format=\"avrodata\")";
    snk = (TextFileSink) FlumeBuilder.buildSink(new Context(), s);
    assertTrue(snk.getFormat() instanceof AvroDataFileOutputFormat);


    s = "text(\"filename\", format=\"avrojson\")";
    snk = (TextFileSink) FlumeBuilder.buildSink(new Context(), s);
    assertTrue(snk.getFormat() instanceof AvroJsonOutputFormat);
  }

View Full Code Here


    try {
      // make sure the sink specified is parsable and instantiable.


      // TODO the first arg should be physical node name
      Context ctx = new LogicalNodeContext(logicalNode, logicalNode);
      FlumeBuilder.buildSink(ctx, sink);
      FlumeBuilder.buildSource(ctx, source);
    } catch (Exception e) {
      throw new IllegalArgumentException(
          "Attempted to write an invalid sink/source: " + e.getMessage(), e);

View Full Code Here

  }


  @Test
  public void testSimpleBuilder() throws FlumeSpecException {
    String spec = "{benchinject => null}";
    FlumeBuilder.buildSink(new Context(), spec);
  }

View Full Code Here

  }


  @Test
  public void testSimpleBuilder2() throws FlumeSpecException {
    String spec = "{benchreport(\"report\") => null}";
    FlumeBuilder.buildSink(new Context(), spec);
  }

View Full Code Here

  }


  @Test
  public void testBuildReportSink2() throws FlumeSpecException {
    String spec = "{benchreport(\"report\", \"text(\\\"test\\\")\") => null } ";
    FlumeBuilder.buildSink(new Context(), spec);
  }

View Full Code Here

  }


  @Test
  public void testBuildInjectDeco() throws FlumeSpecException {
    String spec = "{benchinject(\"report\") => null}";
    FlumeBuilder.buildSink(new Context(), spec);
  }

View Full Code Here

        "{lazyOpen => counter(\"%s\")}");
    LOG.info(agent1);
    assertEquals(
        "failChain(\"{lazyOpen => counter(\\\"%s\\\")}\",\"collector 2\",\"collector 3\",\"collector 1\")",
        agent1);
    CompositeSink snk1 = new CompositeSink(new Context(), agent1);
    snk1.open();
    snk1.close();


    String agent2 = am.getFailChainSinkSpec("agent2",
        "{lazyOpen => counter(\"%s\")}");
    LOG.info(agent2);
    assertEquals(
        "failChain(\"{lazyOpen => counter(\\\"%s\\\")}\",\"collector 1\",\"collector 4\",\"collector 5\")",
        agent2);
    CompositeSink snk2 = new CompositeSink(new Context(), agent2);
    snk2.open();
    snk2.close();


    String agent3 = am.getFailChainSinkSpec("agent3",
        "{lazyOpen => counter(\"%s\")}");
    LOG.info(agent3);
    assertEquals(
        "failChain(\"{lazyOpen => counter(\\\"%s\\\")}\",\"collector 1\",\"collector 2\",\"collector 5\")",
        agent3);
    CompositeSink snk3 = new CompositeSink(new Context(), agent3);
    snk3.open();
    snk3.close();


  }

View Full Code Here


  @Test
  public void testBatchBuilder() throws FlumeSpecException {
    String cfg = " { batch(10) => {unbatch => counter(\"cnt\") }}";
    @SuppressWarnings("unused")
    EventSink sink = FlumeBuilder.buildSink(new Context(), cfg);
  }

View Full Code Here


  @Test
  public void testGzipBuilder() throws FlumeSpecException {
    String cfg = " { gzip => {gunzip => counter(\"cnt\") }}";
    @SuppressWarnings("unused")
    EventSink sink = FlumeBuilder.buildSink(new Context(), cfg);


  }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of com.cloudera.flume.conf.Context

com.cloudera.flume.agent.AgentFailChainSink

com.cloudera.flume.agent.AgentSink

com.cloudera.flume.agent.diskfailover.NaiveFileFailoverManager

com.cloudera.flume.agent.diskfailover.TestDiskFailoverBehavior

com.cloudera.flume.agent.diskfailover.TestDiskFailoverDeco

com.cloudera.flume.agent.durability.TestConcurrentWALMan

com.cloudera.flume.agent.durability.TestNaiveFileWALDeco

com.cloudera.flume.agent.durability.TestNaiveFileWALManagerConcurrently

com.cloudera.flume.agent.FlumeNode

com.cloudera.flume.agent.LogicalNodeManager

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.