Examples of com.cloudera.flume.conf.FlumeConfigData

com.cloudera.flume.conf.FlumeConfigData
Stores configuration data for a Flume agent. Since we use multiple RPC frameworks with slightly different data formats, this class provides translation to and from those transport-specific types.

  public void testBulkSet() throws IOException {
    FlowConfigManager fcm = createSimple();


    Map<String, FlumeConfigData> configs = new HashMap<String, FlumeConfigData>() {
      {
        put("node1", new FlumeConfigData(1, "null", "null", 1, 1, "flow1"));
        put("node21", new FlumeConfigData(1, "null", "null", 1, 1, "flow2"));
        put("node22", new FlumeConfigData(1, "null", "null", 1, 1, "flow2"));
        put("node31", new FlumeConfigData(1, "null", "null", 1, 1, "flow3"));
        put("node32", new FlumeConfigData(1, "null", "null", 1, 1, "flow3"));
        put("node33", new FlumeConfigData(1, "null", "null", 1, 1, "flow3"));
      }
    };


    fcm.setBulkConfig(configs);

View Full Code Here


  public static FlumeConfigData configFromThrift(ThriftFlumeConfigData in) {
    if (in == null) {
      return null;
    }
    FlumeConfigData out = new FlumeConfigData();
    out.timestamp = in.timestamp;
    out.sourceConfig = in.sourceConfig;
    out.sinkConfig = in.sinkConfig;
    out.sourceVersion = in.sourceVersion;
    out.sinkVersion = in.sinkVersion;

View Full Code Here

    // update the configurations
    trans.updateAll();
    int port = FlumeConfiguration.get().getCollectorPort();


    // check if translated
    FlumeConfigData transData = trans.getConfig("bar");
    assertEquals("null", transData.getSourceConfig());
    assertEquals("rpcSink( \"foo\", " + port + " )", transData.getSinkConfig());


    FlumeConfigData transData2 = trans.getConfig("foo");
    assertEquals("rpcSource( " + port + " )", transData2.getSourceConfig());
    assertEquals("null", transData2.getSinkConfig());


    // self is same as translated
    FlumeConfigData selfData = self.getConfig("bar");
    assertEquals("null", selfData.getSourceConfig());
    assertEquals("rpcSink( \"foo\", " + port + " )", selfData.getSinkConfig());


    FlumeConfigData selfData2 = self.getConfig("foo");
    assertEquals("rpcSource( " + port + " )", selfData2.getSourceConfig());
    assertEquals("null", selfData2.getSinkConfig());


    // original is the user entered values
    FlumeConfigData origData = parent.getConfig("bar");
    assertEquals("null", origData.getSourceConfig());
    assertEquals("logicalSink(\"foo\")", origData.getSinkConfig());


    FlumeConfigData origData2 = parent.getConfig("foo");
    assertEquals("logicalSource", origData2.getSourceConfig());
    assertEquals("null", origData2.getSinkConfig());
  }

View Full Code Here


    // now set configs
    trans.setConfig("foo", DEFAULTFLOW, "logicalSource", "null");
    trans.setConfig("bar", DEFAULTFLOW, "null", "logicalSink(\"foo\")");


    FlumeConfigData transData = trans.getConfig("bar");
    assertEquals("null", transData.getSourceConfig());
    assertTrue(transData.getSinkConfig().contains("fail"));


    FlumeConfigData transData2 = trans.getConfig("foo");
    assertTrue(transData2.getSourceConfig().contains("fail"));
    assertEquals("null", transData2.getSinkConfig());


  }

View Full Code Here

    LOG.info("Full Translation: " + trans);
    LOG.info("logical Translation: " + logical);
    LOG.info("failover Translation: " + failover);


    // check the interesting configurations
    FlumeConfigData transData = trans.getConfig("bar");
    assertEquals("null", transData.getSourceConfig());
    String lh = NetUtils.localhost();
    String translated = "< { lazyOpen => rpcSink( \"" + lh + "\", 35853 ) } ? "
        + "< { lazyOpen => rpcSink( \"" + lh + "\", 35855 ) } ? "
        + "< { lazyOpen => rpcSink( \"" + lh + "\", 35854 ) } ? null > > >";
    assertEquals(translated, transData.getSinkConfig());


    FlumeConfigData transCollData = trans.getConfig("foo");
    assertEquals("rpcSource( 35853 )", transCollData.getSourceConfig());
    assertEquals("null", transCollData.getSinkConfig());


    // intermediate data
    FlumeConfigData failData = failover.getConfig("bar");
    assertEquals("null", failData.getSourceConfig());
    String failTranslated = "< { lazyOpen => logicalSink( \"foo\" ) } ? "
        + "< { lazyOpen => logicalSink( \"foo3\" ) } ? "
        + "< { lazyOpen => logicalSink( \"foo2\" ) } ? null > > >";
    assertEquals(failTranslated, failData.getSinkConfig());


  }

View Full Code Here

    // refresh all the configs so that the logical translations kick in.
    trans.updateAll();


    // check to see physical translations.
    for (String[] objs : lists) {
      FlumeConfigData fcd = trans.getConfig(objs[0]);
      LOG.info(objs[0] + "  " + fcd);
      assertEquals(objs[3], fcd.getSourceConfig());
      LOG.info(fcd + " contains " + objs[4] + "?");
      assertTrue(fcd.getSinkConfig().contains(objs[4]));
      LOG.info(fcd + " does not contain logicalSink?");
      assertFalse(fcd.getSinkConfig().contains("logicalSink"));
    }
  }

View Full Code Here

    setupCollectorAgentConfigs();


    // update the configurations
    trans.updateAll();


    FlumeConfigData transData0 = trans.getConfig("bar");
    assertEquals("null", transData0.getSourceConfig());
    String lh = NetUtils.localhost();
    String translated = "< { lazyOpen => rpcSink( \"" + lh + "\", 35853 ) } ? "
        + "< { lazyOpen => rpcSink( \"" + lh + "\", 35855 ) } ? "
        + "< { lazyOpen => rpcSink( \"" + lh + "\", 35854 ) } ? null > > >";
    assertEquals(translated, transData0.getSinkConfig());


    trans.updateAll();


    FlumeConfigData transData = trans.getConfig("bar");
    assertEquals("null", transData.getSourceConfig());
    assertEquals(translated, transData.getSinkConfig());


    FlumeConfigData transCollData = trans.getConfig("foo");
    assertEquals("rpcSource( 35853 )", transCollData.getSourceConfig());
    assertEquals("null", transCollData.getSinkConfig());


    trans.updateAll();


    transData = trans.getConfig("bar");
    assertEquals("null", transData.getSourceConfig());
    assertEquals(translated, transData.getSinkConfig());


    transCollData = trans.getConfig("foo");
    assertEquals("rpcSource( 35853 )", transCollData.getSourceConfig());
    assertEquals("null", transCollData.getSinkConfig());


    trans.updateAll();


    LOG.info("Full Translation: " + trans);
    LOG.info("logical Translation: " + logical);
    LOG.info("failover Translation: " + failover);


    transData = trans.getConfig("bar");
    assertEquals("null", transData.getSourceConfig());
    assertEquals(translated, transData.getSinkConfig());


    transCollData = trans.getConfig("foo");
    assertEquals("rpcSource( 35853 )", transCollData.getSourceConfig());
    assertEquals("null", transCollData.getSinkConfig());


    // intermediate data
    FlumeConfigData failData = failover.getConfig("bar");
    assertEquals("null", failData.getSourceConfig());
    String failTranslated = "< { lazyOpen => logicalSink( \"foo\" ) } ? "
        + "< { lazyOpen => logicalSink( \"foo3\" ) } ? "
        + "< { lazyOpen => logicalSink( \"foo2\" ) } ? null > > >";
    assertEquals(failTranslated, failData.getSinkConfig());
  }

View Full Code Here


    LOG.info("Full Translation: " + trans);
    LOG.info("logical Translation: " + logical);
    LOG.info("failover Translation: " + failover);


    FlumeConfigData transData0 = trans.getConfig("bar");
    assertEquals("null", transData0.getSourceConfig());
    String lh = NetUtils.localhost();
    String translated = "< { lazyOpen => rpcSink( \"" + lh + "\", 35853 ) } ? "
        + "< { lazyOpen => rpcSink( \"" + lh + "\", 35855 ) } ? "
        + "< { lazyOpen => rpcSink( \"" + lh + "\", 35854 ) } ? null > > >";
    assertEquals(translated, transData0.getSinkConfig());


    trans.updateAll();


    FlumeConfigData transData = trans.getConfig("bar");
    long barVer = transData.getTimestamp();
    assertEquals("null", transData.getSourceConfig());
    assertEquals(translated, transData.getSinkConfig());


    FlumeConfigData transCollData = trans.getConfig("foo");
    long fooVer = transCollData.getTimestamp();
    assertEquals("rpcSource( 35853 )", transCollData.getSourceConfig());
    assertEquals("null", transCollData.getSinkConfig());


    trans.updateAll();


    transData = trans.getConfig("bar");
    assertEquals(barVer, transData.getTimestamp());
    assertEquals("null", transData.getSourceConfig());
    assertEquals(translated, transData.getSinkConfig());


    transCollData = trans.getConfig("foo");
    assertEquals(fooVer, transCollData.getTimestamp());
    assertEquals("rpcSource( 35853 )", transCollData.getSourceConfig());
    assertEquals("null", transCollData.getSinkConfig());


    trans.updateAll(); // TODO (jon) Ideally, this shouldn't be necessary


    LOG.info("Full Translation: " + trans);
    LOG.info("Logical Translation: " + logical);
    LOG.info("Failover Translation: " + failover);


    transData = trans.getConfig("bar");
    assertEquals(barVer, transData.getTimestamp());
    assertEquals("null", transData.getSourceConfig());
    assertEquals(translated, transData.getSinkConfig());


    transCollData = trans.getConfig("foo");
    assertEquals(fooVer, transCollData.getTimestamp());
    assertEquals("rpcSource( 35853 )", transCollData.getSourceConfig());
    assertEquals("null", transCollData.getSinkConfig());


    // intermediate data
    FlumeConfigData failData = failover.getConfig("bar");
    assertEquals("null", failData.getSourceConfig());
    String failTranslated = "< { lazyOpen => logicalSink( \"foo\" ) } ? "
        + "< { lazyOpen => logicalSink( \"foo3\" ) } ? "
        + "< { lazyOpen => logicalSink( \"foo2\" ) } ? null > > >";
    assertEquals(failTranslated, failData.getSinkConfig());
  }

View Full Code Here


    LOG.info("Full Translation: " + trans);
    LOG.info("logical Translation: " + logical);
    LOG.info("failover Translation: " + failover);


    FlumeConfigData transData = trans.getConfig("bar");
    assertEquals("null", transData.getSourceConfig());
    String lh = NetUtils.localhost();
    String translated = "< { lazyOpen => rpcSink( \"" + lh + "\", 35853 ) } ? "
        + "< { lazyOpen => rpcSink( \"" + lh + "\", 35854 ) } ? null > >";
    assertEquals(translated, transData.getSinkConfig());


    FlumeConfigData transCollData = trans.getConfig("foo");
    assertEquals("rpcSource( 35853 )", transCollData.getSourceConfig());
    assertEquals("null", transCollData.getSinkConfig());


    assertEquals(fooVer, trans.getConfig("foo").getTimestamp());
    assertEquals(foo2Ver, trans.getConfig("foo2").getTimestamp());
    assertNotSame(foo3Ver, trans.getConfig("foo3").getTimestamp());
    foo3Ver = trans.getConfig("foo3").getTimestamp();
    assertNotSame(barVer, trans.getConfig("bar").getTimestamp());
    barVer = trans.getConfig("bar").getTimestamp();


    // intermediate data
    FlumeConfigData failData = failover.getConfig("bar");
    assertEquals("null", failData.getSourceConfig());
    String failTranslated = "< { lazyOpen => logicalSink( \"foo\" ) } ? "
        + "< { lazyOpen => logicalSink( \"foo2\" ) } ? null > >";
    assertEquals(failTranslated, failData.getSinkConfig());


    assertEquals(fooVer, trans.getConfig("foo").getTimestamp());
    assertEquals(foo2Ver, trans.getConfig("foo2").getTimestamp());
    assertEquals(foo3Ver, trans.getConfig("foo3").getTimestamp());
    assertEquals(barVer, trans.getConfig("bar").getTimestamp());

View Full Code Here

    // Look, no explicit update call!


    // check new config
    Map<String, FlumeConfigData> xcfgs2 = flumeMaster.getSpecMan()
        .getTranslatedConfigs();
    FlumeConfigData agentFcd2 = xcfgs2.get("agent");
    String ans2 = "< { lazyOpen => fail( \"logicalSink( \\\"node4\\\" )\" ) } ?"
        + " < { lazyOpen => fail( \"logicalSink( \\\"node2\\\" )\" ) } ?"
        + " < { lazyOpen => fail( \"logicalSink( \\\"node1\\\" )\" ) } ? null > > >";
    assertEquals(ans2, agentFcd2.sinkConfig);

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of com.cloudera.flume.conf.FlumeConfigData

com.cloudera.flume.agent.DirectMasterRPC

com.cloudera.flume.agent.diskfailover.TestDiskFailoverBehavior

com.cloudera.flume.agent.LivenessManager

com.cloudera.flume.agent.LogicalNode

com.cloudera.flume.agent.LogicalNodeManager

com.cloudera.flume.agent.TestAvroMultiMasterRPC$MockAvroServer

com.cloudera.flume.agent.TestFlumeNode

com.cloudera.flume.agent.TestLogicalNode

com.cloudera.flume.agent.TestRPCMechanisms

com.cloudera.flume.agent.TestRPCMechanisms$MockAvroServer

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.