Examples of FlumeConfiguration

com.cloudera.flume.conf.FlumeConfiguration
This is the configuration file for flume. It uses hadoop's xml schema for properties. The default configuration is in $FLUME_HOME/flume-conf.xml and is a singleton. NOTE: The logging system gets configuration information from this file and thus depends on this being initialized. So no logging events are allowed when this is being initialized. If logger is used, an infinite recursion happens and causes a stack overflow.
org.apache.flume.conf.FlumeConfiguration

FlumeConfiguration is an in memory representation of the hierarchical configuration namespace required by the PropertiesFileConfigurationProvider. This class is instantiated with a properties object which is parsed to construct the hierarchy in memory. Once the entire set of properties have been parsed and populated, a validation routine is run that identifies and removes invalid components.
@see org.apache.flume.conf.properties.PropertiesFileConfigurationProvider

Examples of com.cloudera.flume.conf.FlumeConfiguration

  public void testZKBackedConfigStore() throws IOException,
      InterruptedException {
    for (int i = 0; i < 10; ++i) {
      LOG.info("Opening ZK, attempt " + i);
      File tmp = FileUtil.mktempdir();
      FlumeConfiguration cfg = FlumeConfiguration.createTestableConfiguration();
      cfg.set(FlumeConfiguration.MASTER_ZK_LOGDIR, tmp.getAbsolutePath());
      cfg.set(FlumeConfiguration.MASTER_ZK_SERVERS, "localhost:2181:3181:4181");
      ZooKeeperService.getAndInit(cfg);


      ZooKeeperConfigStore store = new ZooKeeperConfigStore();
      store.init();
      ConfigManager manager = new ConfigManager(store);
      manager.setConfig("foo", "my-test-flow", "null", "console");
      FlumeConfigData data = manager.getConfig("foo");
      assertEquals(data.getSinkConfig(), "console");
      assertEquals(data.getSourceConfig(), "null");
      store.shutdown();


      store = new ZooKeeperConfigStore();
      store.init();
      manager = new ConfigManager(store);
      data = manager.getConfig("foo");
      assertEquals(data.getSinkConfig(), "console");
      assertEquals(data.getSourceConfig(), "null");


      Map<String, FlumeConfigData> cfgs = new HashMap<String, FlumeConfigData>();
      String defaultFlowName = cfg.getDefaultFlowName();
      cfgs.put("bulk1", new FlumeConfigData(0, "s1", "sk1",
          LogicalNode.VERSION_INFIMUM, LogicalNode.VERSION_INFIMUM,
          "my-test-flow"));
      cfgs.put("bulk2", new FlumeConfigData(0, "s2", "sk2",
          LogicalNode.VERSION_INFIMUM, LogicalNode.VERSION_INFIMUM,

View Full Code Here

Examples of com.cloudera.flume.conf.FlumeConfiguration

   */
  @Test
  public void testZKBackedConfigStoreNodes() throws IOException,
      InterruptedException {
    File tmp = FileUtil.mktempdir();
    FlumeConfiguration cfg = FlumeConfiguration.createTestableConfiguration();
    cfg.set(FlumeConfiguration.MASTER_ZK_LOGDIR, tmp.getAbsolutePath());
    cfg.set(FlumeConfiguration.MASTER_ZK_SERVERS, "localhost:2181:3181:4181");
    ZooKeeperService.getAndInit(cfg);


    ZooKeeperConfigStore store = new ZooKeeperConfigStore();
    store.init();

View Full Code Here

Examples of com.cloudera.flume.conf.FlumeConfiguration

  /**
   * Test that watches are fired correctly for logical nodes
   */
  @Test
  public void testZBCSLogicalWatches() throws IOException, InterruptedException {
    FlumeConfiguration cfg = FlumeConfiguration.createTestableConfiguration();
    cfg.set(FlumeConfiguration.MASTER_ZK_SERVERS, "localhost:2181:3181:4181");
    File tmp = FileUtil.mktempdir();
    cfg.set(FlumeConfiguration.MASTER_ZK_LOGDIR, tmp.getAbsolutePath());
    cfg.setBoolean(FlumeConfiguration.MASTER_ZK_USE_EXTERNAL, false);
    ZooKeeperService.getAndInit(cfg);


    ZooKeeperConfigStore store = new ZooKeeperConfigStore();
    store.init();
    ZooKeeperConfigStore store2 = new ZooKeeperConfigStore();

View Full Code Here

Examples of com.cloudera.flume.conf.FlumeConfiguration

  /**
   * Test that watches are fired correctly for ChokeMap.
   */
  @Test
  public void testZBCSChokeWatches() throws IOException, InterruptedException {
    FlumeConfiguration cfg = FlumeConfiguration.createTestableConfiguration();
    cfg.set(FlumeConfiguration.MASTER_ZK_SERVERS, "localhost:2181:3181:4181");
    File tmp = FileUtil.mktempdir();
    cfg.set(FlumeConfiguration.MASTER_ZK_LOGDIR, tmp.getAbsolutePath());
    cfg.setBoolean(FlumeConfiguration.MASTER_ZK_USE_EXTERNAL, false);
    ZooKeeperService.getAndInit(cfg);


    ZooKeeperConfigStore store = new ZooKeeperConfigStore();
    store.init();
    ZooKeeperConfigStore store2 = new ZooKeeperConfigStore();

View Full Code Here

Examples of com.cloudera.flume.conf.FlumeConfiguration

   * Test disconnection
   */
  @Test
  public void testZBCSLoseZKCnxn() throws IOException, InterruptedException {
    File tmp = FileUtil.mktempdir();
    FlumeConfiguration cfg = FlumeConfiguration.createTestableConfiguration();
    cfg.set(FlumeConfiguration.MASTER_ZK_LOGDIR, tmp.getAbsolutePath());
    cfg.set(FlumeConfiguration.MASTER_ZK_SERVERS, "localhost:2181:3181:4181");
    ZooKeeperService.getAndInit(cfg);


    ZooKeeperConfigStore store = new ZooKeeperConfigStore();
    store.init();
    store.setConfig("foo", "my-test-flow", "fab", "fat");


    ZooKeeperService.get().shutdown();
    Thread.sleep(30 * 1000);
    IOException ex = null;


    try {
      store.setConfig("foo", cfg.getDefaultFlowName(), "bar", "baz");
    } catch (IOException e) {
      ex = e;
    }
    assertNotNull("Expected IOException not thrown - still connected to ZK?",
        ex);

View Full Code Here

Examples of com.cloudera.flume.conf.FlumeConfiguration

   */
  @Test
  public void testUnmapAllNodes() throws IOException, InterruptedException {


    File tmp = FileUtil.mktempdir();
    FlumeConfiguration cfg = FlumeConfiguration.createTestableConfiguration();
    cfg.set(FlumeConfiguration.MASTER_ZK_LOGDIR, tmp.getAbsolutePath());
    cfg.set(FlumeConfiguration.MASTER_ZK_SERVERS, "localhost:2181:3181:4181");
    cfg.setInt(FlumeConfiguration.MASTER_SERVER_ID, 0);
    ZooKeeperService.getAndInit(cfg);
    ZooKeeperConfigStore store = new ZooKeeperConfigStore();
    store.init();


    ConfigManager manager = new ConfigManager(store);

View Full Code Here

Examples of com.cloudera.flume.conf.FlumeConfiguration

   */
  @Test
  public void testVersionIncrement() throws IOException, InterruptedException,
      KeeperException {
    File tmp = FileUtil.mktempdir();
    FlumeConfiguration cfg = FlumeConfiguration.createTestableConfiguration();
    cfg.set(FlumeConfiguration.MASTER_ZK_LOGDIR, tmp.getAbsolutePath());
    cfg.set(FlumeConfiguration.MASTER_ZK_SERVERS, "localhost:2181:3181:4181");
    cfg.setInt(FlumeConfiguration.MASTER_SERVER_ID, 0);


    ZooKeeperService zk = new ZooKeeperService();
    zk.init(cfg);


    ZooKeeperConfigStore store = new ZooKeeperConfigStore(zk);
    store.init();
    String defaultFlowName = cfg.getDefaultFlowName();
    store.setConfig("foo", defaultFlowName, "null", "baz");
    store.setConfig("foo2", defaultFlowName, "null", "baz");
    store.setConfig("foo3", defaultFlowName, "null", "baz");


    ZKClient client = zk.createClient();

View Full Code Here

Examples of com.cloudera.flume.conf.FlumeConfiguration

  @Test
  @Ignore("Timing issue prevents this succeeding on Hudson")
  public void testLostSessionOK() throws IOException, InterruptedException,
      KeeperException {
    File tmp = FileUtil.mktempdir();
    FlumeConfiguration cfg = FlumeConfiguration.createTestableConfiguration();
    cfg.set(FlumeConfiguration.MASTER_ZK_LOGDIR, tmp.getAbsolutePath());
    cfg.set(FlumeConfiguration.MASTER_ZK_SERVERS, "localhost:2181:3181:4181");
    cfg.setInt(FlumeConfiguration.MASTER_SERVER_ID, 0);


    ZooKeeperService zk = new ZooKeeperService();
    zk.init(cfg);


    ZooKeeperConfigStore store = new ZooKeeperConfigStore(zk);
    store.init();
    String defaultFlowName = cfg.getDefaultFlowName();
    store.setConfig("foo", defaultFlowName, "bar", "baz");
    long sessionid = store.client.zk.getSessionId();
    byte[] sessionpass = store.client.zk.getSessionPasswd();


    // Force session expiration

View Full Code Here

Examples of com.cloudera.flume.conf.FlumeConfiguration

   * @throws InterruptedException
   */
  @Test
  public void testBZip2Codec() throws IOException, InterruptedException {
    // set the output format.
    FlumeConfiguration conf = FlumeConfiguration.get();
    conf.set(FlumeConfiguration.COLLECTOR_OUTPUT_FORMAT, "syslog");
    conf.set(FlumeConfiguration.COLLECTOR_DFS_COMPRESS_CODEC, "BZip2Codec");


    // build a sink that outputs to that format.
    File f = FileUtil.mktempdir();
    SinkBuilder builder = EscapedCustomDfsSink.builder();
    EventSink snk = builder.build(new Context(), "file:///" + f.getPath()

View Full Code Here

Examples of com.cloudera.flume.conf.FlumeConfiguration

   * @throws InterruptedException
   */
  @Test
  public void testGzipOutputFormat() throws IOException, InterruptedException {
    // set the output format.
    FlumeConfiguration conf = FlumeConfiguration.get();
    conf.set(FlumeConfiguration.COLLECTOR_OUTPUT_FORMAT, "syslog");
    conf.set(FlumeConfiguration.COLLECTOR_DFS_COMPRESS_GZIP, "true");


    // build a sink that outputs to that format.
    File f = FileUtil.mktempdir();
    SinkBuilder builder = EscapedCustomDfsSink.builder();
    EventSink snk = builder.build(new Context(), "file:///" + f.getPath()

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.