Examples of FlumeConfiguration

com.cloudera.flume.conf.FlumeConfiguration
This is the configuration file for flume. It uses hadoop's xml schema for properties. The default configuration is in $FLUME_HOME/flume-conf.xml and is a singleton. NOTE: The logging system gets configuration information from this file and thus depends on this being initialized. So no logging events are allowed when this is being initialized. If logger is used, an infinite recursion happens and causes a stack overflow.
org.apache.flume.conf.FlumeConfiguration

FlumeConfiguration is an in memory representation of the hierarchical configuration namespace required by the PropertiesFileConfigurationProvider. This class is instantiated with a properties object which is parsed to construct the hierarchy in memory. Once the entire set of properties have been parsed and populated, a validation routine is run that identifies and removes invalid components.
@see org.apache.flume.conf.properties.PropertiesFileConfigurationProvider

Examples of org.apache.flume.conf.FlumeConfiguration

      in.close();
    }
  }


  protected void verifyProperties(AbstractConfigurationProvider cp) {
    FlumeConfiguration configuration = cp.getFlumeConfiguration();
    Assert.assertNotNull(configuration);


    /*
     * Test the known errors in the file
     */
    List<String> expected = Lists.newArrayList();
    expected.add("host5 CONFIG_ERROR");
    expected.add("host5 INVALID_PROPERTY");
    expected.add("host4 CONFIG_ERROR");
    expected.add("host4 CONFIG_ERROR");
    expected.add("host4 PROPERTY_VALUE_NULL");
    expected.add("host4 PROPERTY_VALUE_NULL");
    expected.add("host4 PROPERTY_VALUE_NULL");
    expected.add("host4 AGENT_CONFIGURATION_INVALID");
    expected.add("ch2 ATTRS_MISSING");
    expected.add("host3 CONFIG_ERROR");
    expected.add("host3 PROPERTY_VALUE_NULL");
    expected.add("host3 AGENT_CONFIGURATION_INVALID");
    expected.add("host2 PROPERTY_VALUE_NULL");
    expected.add("host2 AGENT_CONFIGURATION_INVALID");
    List<String> actual = Lists.newArrayList();
    for (FlumeConfigurationError error : configuration
      .getConfigurationErrors()) {
      actual.add(error.getComponentName() + " "
          + error.getErrorType().toString());
    }
    Collections.sort(expected);
    Collections.sort(actual);
    Assert.assertEquals(expected, actual);


    FlumeConfiguration.AgentConfiguration agentConfiguration = configuration
        .getConfigurationFor("host1");
    Assert.assertNotNull(agentConfiguration);


    Set<String> sources = Sets.newHashSet("source1");
    Set<String> sinks = Sets.newHashSet("sink1");

View Full Code Here

Examples of org.apache.flume.conf.FlumeConfiguration

  @Test
  public void testPolling() throws Exception {
    es.awaitEvent();
    es.reset();


    FlumeConfiguration fc = cp.getFlumeConfiguration();
    Assert.assertTrue(fc.getConfigurationErrors().isEmpty());
    AgentConfiguration ac = fc.getConfigurationFor(AGENT_NAME);
    Assert.assertNull(ac);


    addData();
    es.awaitEvent();
    es.reset();

View Full Code Here

Examples of org.apache.flume.conf.FlumeConfiguration


  protected abstract FlumeConfiguration getFlumeConfiguration();


  public MaterializedConfiguration getConfiguration() {
    MaterializedConfiguration conf = new SimpleMaterializedConfiguration();
    FlumeConfiguration fconfig = getFlumeConfiguration();
    AgentConfiguration agentConf = fconfig.getConfigurationFor(getAgentName());
    if (agentConf != null) {
      Map<String, ChannelComponent> channelComponentMap = Maps.newHashMap();
      Map<String, SourceRunner> sourceRunnerMap = Maps.newHashMap();
      Map<String, SinkRunner> sinkRunnerMap = Maps.newHashMap();
      try {

View Full Code Here

Examples of org.apache.flume.conf.FlumeConfiguration

    BufferedReader reader = null;
    try {
      reader = new BufferedReader(new FileReader(file));
      Properties properties = new Properties();
      properties.load(reader);
      return new FlumeConfiguration(toMap(properties));
    } catch (IOException ex) {
      LOGGER.error("Unable to load file:" + file
          + " (I/O failure) - Exception follows.", ex);
    } finally {
      if (reader != null) {
        try {
          reader.close();
        } catch (IOException ex) {
          LOGGER.warn(
              "Unable to close file reader for file: " + file, ex);
        }
      }
    }
    return new FlumeConfiguration(new HashMap<String, String>());
  }

View Full Code Here

Examples of org.apache.flume.conf.FlumeConfiguration


  protected abstract FlumeConfiguration getFlumeConfiguration();


  public MaterializedConfiguration getConfiguration() {
    MaterializedConfiguration conf = new SimpleMaterializedConfiguration();
    FlumeConfiguration fconfig = getFlumeConfiguration();
    AgentConfiguration agentConf = fconfig.getConfigurationFor(getAgentName());
    if (agentConf != null) {
      Map<String, ChannelComponent> channelComponentMap = Maps.newHashMap();
      Map<String, SourceRunner> sourceRunnerMap = Maps.newHashMap();
      Map<String, SinkRunner> sinkRunnerMap = Maps.newHashMap();
      try {

View Full Code Here

Examples of org.apache.flume.conf.FlumeConfiguration

    BufferedReader reader = null;
    try {
      reader = new BufferedReader(new FileReader(file));
      Properties properties = new Properties();
      properties.load(reader);
      return new FlumeConfiguration(toMap(properties));
    } catch (IOException ex) {
      LOGGER.error("Unable to load file:" + file
          + " (I/O failure) - Exception follows.", ex);
    } finally {
      if (reader != null) {
        try {
          reader.close();
        } catch (IOException ex) {
          LOGGER.warn(
              "Unable to close file reader for file: " + file, ex);
        }
      }
    }
    return new FlumeConfiguration(new HashMap<String, String>());
  }

View Full Code Here

Examples of org.apache.flume.conf.FlumeConfiguration

    this.properties = properties;
  }


  @Override
  protected FlumeConfiguration getFlumeConfiguration() {
    return new FlumeConfiguration(properties);
  }

View Full Code Here

Examples of org.apache.flume.conf.FlumeConfiguration

      this.properties = properties;
    }


    @Override
    protected FlumeConfiguration getFlumeConfiguration() {
      return new FlumeConfiguration(properties);
    }

View Full Code Here

Examples of org.apache.flume.conf.FlumeConfiguration

  }


  @Test
  public void testPropertyRead() throws Exception {


    FlumeConfiguration configuration = provider.getFlumeConfiguration();
    Assert.assertNotNull(configuration);


    /*
     * Test the known errors in the file
     */
    List<String> expected = Lists.newArrayList();
    expected.add("host5 CONFIG_ERROR");
    expected.add("host5 INVALID_PROPERTY");
    expected.add("host4 CONFIG_ERROR");
    expected.add("host4 CONFIG_ERROR");
    expected.add("host4 PROPERTY_VALUE_NULL");
    expected.add("host4 PROPERTY_VALUE_NULL");
    expected.add("host4 PROPERTY_VALUE_NULL");
    expected.add("host4 AGENT_CONFIGURATION_INVALID");
    expected.add("ch2 ATTRS_MISSING");
    expected.add("host3 CONFIG_ERROR");
    expected.add("host3 PROPERTY_VALUE_NULL");
    expected.add("host3 AGENT_CONFIGURATION_INVALID");
    expected.add("host2 PROPERTY_VALUE_NULL");
    expected.add("host2 AGENT_CONFIGURATION_INVALID");
    List<String> actual = Lists.newArrayList();
    for(FlumeConfigurationError error : configuration.getConfigurationErrors()) {
      actual.add(error.getComponentName() + " " + error.getErrorType().toString());
    }
    Collections.sort(expected);
    Collections.sort(actual);
    Assert.assertEquals(expected, actual);




    AgentConfiguration agentConfiguration =
        configuration.getConfigurationFor("host1");
    Assert.assertNotNull(agentConfiguration);




    LOGGER.info(agentConfiguration.getPrevalidationConfig());
    LOGGER.info(agentConfiguration.getPostvalidationConfig());

View Full Code Here

Examples of org.apache.flume.conf.FlumeConfiguration

    private final SinkFactory sinkFactory = new DefaultSinkFactory();


    public NodeConfiguration load(final String name, final Properties props,
                                  final NodeConfigurationAware configurationAware) {
        final NodeConfiguration conf = new SimpleNodeConfiguration();
        FlumeConfiguration fconfig;
        try {
            fconfig = new FlumeConfiguration(props);
            final List<FlumeConfigurationError> errors = fconfig.getConfigurationErrors();
            if (errors.size() > 0) {
                boolean isError = false;
                for (final FlumeConfigurationError error : errors) {
                    final StringBuilder sb = new StringBuilder();
                    sb.append("Component: ").append(error.getComponentName()).append(" ");
                    sb.append("Key: ").append(error.getKey()).append(" ");
                    sb.append(error.getErrorType().name()).append(" - ").append(error.getErrorType().getError());
                    switch (error.getErrorOrWarning()) {
                        case ERROR:
                            isError = true;
                            LOGGER.error(sb.toString());
                            break;
                        case WARNING:
                            LOGGER.warn(sb.toString());
                            break;
                    }
                }
                if (isError) {
                    throw new ConfigurationException("Unable to configure Flume due to errors");
                }
            }
        } catch (final RuntimeException ex) {
            printProps(props);
            throw ex;
        }


        final FlumeConfiguration.AgentConfiguration agentConf = fconfig.getConfigurationFor(name);


        if (agentConf != null) {


            loadChannels(agentConf, conf);
            loadSources(agentConf, conf);

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.