Examples of FlumeConfiguration

com.cloudera.flume.conf.FlumeConfiguration
This is the configuration file for flume. It uses hadoop's xml schema for properties. The default configuration is in $FLUME_HOME/flume-conf.xml and is a singleton. NOTE: The logging system gets configuration information from this file and thus depends on this being initialized. So no logging events are allowed when this is being initialized. If logger is used, an infinite recursion happens and causes a stack overflow.
org.apache.flume.conf.FlumeConfiguration

FlumeConfiguration is an in memory representation of the hierarchical configuration namespace required by the PropertiesFileConfigurationProvider. This class is instantiated with a properties object which is parsed to construct the hierarchy in memory. Once the entire set of properties have been parsed and populated, a validation routine is run that identifies and removes invalid components.
@see org.apache.flume.conf.properties.PropertiesFileConfigurationProvider

Examples of com.cloudera.flume.conf.FlumeConfiguration

   */


  @Test(expected = IOException.class)
  public void testDefaultCodec() throws IOException, InterruptedException {
    // set the output format.
    FlumeConfiguration conf = FlumeConfiguration.get();
    conf.set(FlumeConfiguration.COLLECTOR_OUTPUT_FORMAT, "syslog");
    conf.set(FlumeConfiguration.COLLECTOR_DFS_COMPRESS_CODEC, "DefaultCodec");


    // build a sink that outputs to that format.
    File f = FileUtil.mktempdir();
    SinkBuilder builder = EscapedCustomDfsSink.builder();
    EventSink snk = builder.build(new Context(), "file:///" + f.getPath()

View Full Code Here

Examples of com.cloudera.flume.conf.FlumeConfiguration

    shouldSub = Event.containsTag(path);
  }


  protected Writer openWriter(String p) throws IOException {
    LOG.info("Opening " + p);
    FlumeConfiguration conf = FlumeConfiguration.get();


    Path dstPath = new Path(p);
    FileSystem hdfs = dstPath.getFileSystem(conf);


    Writer w = SequenceFile.createWriter(hdfs, conf, dstPath,

View Full Code Here

Examples of com.cloudera.flume.conf.FlumeConfiguration

      }
    };
  }


  public static void main(String argv[]) {
    FlumeConfiguration conf = FlumeConfiguration.get();
    ThriftRawEventSink sink =
        new ThriftRawEventSink("localhost", conf.getCollectorPort());
    try {
      sink.open();


      for (int i = 0; i < 100; i++) {
        Event e = new EventImpl(("This is a test " + i).getBytes());

View Full Code Here

Examples of com.cloudera.flume.conf.FlumeConfiguration

    writer = null;
  }


  @Override
  public void open() throws IOException {
    FlumeConfiguration conf = FlumeConfiguration.get();
    FileSystem hdfs;


    // use v0.9.1 compression settings
    if (conf.getCollectorDfsCompressGzipStatus()) {
      LOG.warn("Config property "
          + FlumeConfiguration.COLLECTOR_DFS_COMPRESS_GZIP
          + " is deprecated, please use "
          + FlumeConfiguration.COLLECTOR_DFS_COMPRESS_CODEC
          + " set to GzipCodec instead");
      CompressionCodec gzipC = new GzipCodec();
      Compressor gzCmp = gzipC.createCompressor();
      dstPath = new Path(path + gzipC.getDefaultExtension());
      hdfs = dstPath.getFileSystem(conf);
      writer = hdfs.create(dstPath);
      writer = gzipC.createOutputStream(writer, gzCmp);
      LOG.info("Creating HDFS gzip compressed file: " + dstPath.toString());
      return;
    }


    String codecName = conf.getCollectorDfsCompressCodec();
    List<Class<? extends CompressionCodec>> codecs = CompressionCodecFactory
        .getCodecClasses(FlumeConfiguration.get());
    CompressionCodec codec = null;
    ArrayList<String> codecStrs = new ArrayList<String>();
    codecStrs.add("None");

View Full Code Here

Examples of com.cloudera.flume.conf.FlumeConfiguration

  /**
   * Test that full init, create client, shutdown lifecycle works correctly
   */
  @Test
  public void testLifecycle() throws IOException, InterruptedException, KeeperException {
    FlumeConfiguration cfg = FlumeConfiguration.get();
    File tmp = FileUtil.mktempdir();
    cfg.set(FlumeConfiguration.MASTER_ZK_SERVERS, "localhost:2181:3181:4181");
    cfg.set(FlumeConfiguration.MASTER_ZK_LOGDIR, tmp.getAbsolutePath());
    ZooKeeperService svc = new ZooKeeperService();
    svc.init(cfg);
    
    Assert.assertTrue("ZooKeeperService did not initialise", svc.isInitialised());

View Full Code Here

Examples of com.cloudera.flume.conf.FlumeConfiguration

  /**
   * Test that we have correctly disabled the maxClientCnxns bound
   */
  @Test
  public void testMultipleClients() throws IOException, InterruptedException, KeeperException {
    FlumeConfiguration cfg = FlumeConfiguration.get();
    File tmp = FileUtil.mktempdir();
    cfg.set(FlumeConfiguration.MASTER_ZK_SERVERS, "localhost:2181:3181:4181");
    cfg.set(FlumeConfiguration.MASTER_ZK_LOGDIR, tmp.getAbsolutePath());
    ZooKeeperService svc = new ZooKeeperService();
    svc.init(cfg);
    
    Assert.assertTrue("ZooKeeperService did not initialise", svc.isInitialised());

View Full Code Here

Examples of com.cloudera.flume.conf.FlumeConfiguration

      throw new IOException("Waiting for queue element was interrupted! " + e);
    }
  }


  public static void main(String[] argv) {
    FlumeConfiguration conf = FlumeConfiguration.get();
    PrioritizedThriftEventSource src = new PrioritizedThriftEventSource(conf
        .getCollectorPort());


    try {
      src.open();
      Event e;

View Full Code Here

Examples of com.cloudera.flume.conf.FlumeConfiguration

      System.exit(-1);
    }


    String[] args = argv;


    FlumeConfiguration conf = FlumeConfiguration.hardExitLoadConfig();
    int maxTriesPerMin = conf.getMaxRestartsPerMin();


    Watchdog watchdog = new Watchdog(args);
    watchdog.run(maxTriesPerMin);
  }

View Full Code Here

Examples of com.cloudera.flume.conf.FlumeConfiguration

      }
      specman.stop();


      reaper.interrupt();


      FlumeConfiguration cfg = FlumeConfiguration.get();
      if (cfg.getMasterStore().equals(ZK_CFG_STORE)) {
        ZooKeeperService.get().shutdown();
      }


    } catch (IOException e) {
      LOG.error("Exception when shutting down master!", e);

View Full Code Here

Examples of com.cloudera.flume.conf.FlumeConfiguration

    String interactiveprop = System.getProperty("fwdstdin");
    boolean interactive = (interactiveprop != null);


    String[] args = argv;


    FlumeConfiguration conf = FlumeConfiguration.hardExitLoadConfig();
    int maxTriesPerMin = conf.getMaxRestartsPerMin();


    Watchdog watchdog = new Watchdog(args, interactive);
    watchdog.run(maxTriesPerMin);
  }

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.