Examples of org.apache.flume.Source

org.apache.flume.Source

A source generates {@plainlink Event events} and calls methods on theconfigured {@link ChannelProcessor} to persist those events into theconfigured {@linkplain Channel channels}.

Sources are associated with unique {@linkplain NamedComponent names} that canbe used for separating configuration and working namespaces.

No guarantees are given regarding thread safe access.
@see org.apache.flume.Channel @see org.apache.flume.Sink

      InterruptedException {


    Channel channel = new MemoryChannel();
    Configurables.configure(channel, new Context());


    Source source = new SequenceGeneratorSource();
    source.setChannel(channel);


    Sink sink = new NullSink();
    sink.setChannel(channel);


    nodeManager.add(SourceRunner.forSource(source));

View Full Code Here

        for (String sourceName : sources) {
            ComponentConfiguration comp = compMap.get(sourceName);
            if (comp != null) {
                SourceConfiguration config = (SourceConfiguration) comp;


                Source source = sourceFactory.create(comp.getComponentName(), comp.getType());


                Configurables.configure(source, config);
                Set<String> channelNames = config.getChannels();
                List<Channel> channels = new ArrayList<Channel>();
                for (String chName : channelNames) {
                    channels.add(conf.getChannels().get(chName));
                }


                ChannelSelectorConfiguration selectorConfig = config.getSelectorConfiguration();


                ChannelSelector selector = ChannelSelectorFactory.create(channels, selectorConfig);


                ChannelProcessor channelProcessor = new ChannelProcessor(selector);
                Configurables.configure(channelProcessor, config);


                source.setChannelProcessor(channelProcessor);
                conf.getSourceRunners().put(comp.getComponentName(), SourceRunner.forSource(source));
            }
        }
        Map<String, Context> sourceContexts = agentConf.getSourceContext();


        for (String src : sources) {
            Context context = sourceContexts.get(src);
            if (context != null){
                Source source = sourceFactory.create(src, context.getString(BasicConfigurationConstants.CONFIG_TYPE));
                List<Channel> channels = new ArrayList<Channel>();
                Configurables.configure(source, context);
                String[] channelNames = context.getString(BasicConfigurationConstants.CONFIG_CHANNELS).split("\\s+");
                for (String chName : channelNames) {
                    channels.add(conf.getChannels().get(chName));
                }


                Map<String, String> selectorConfig = context.getSubProperties(
                    BasicConfigurationConstants.CONFIG_SOURCE_CHANNELSELECTOR_PREFIX);


                ChannelSelector selector = ChannelSelectorFactory.create(channels, selectorConfig);


                ChannelProcessor channelProcessor = new ChannelProcessor(selector);
                Configurables.configure(channelProcessor, context);


                source.setChannelProcessor(channelProcessor);
                conf.getSourceRunners().put(src, SourceRunner.forSource(source));
            }
        }
    }

View Full Code Here

        for (final String sourceName : sources) {
            final ComponentConfiguration comp = compMap.get(sourceName);
            if (comp != null) {
                final SourceConfiguration config = (SourceConfiguration) comp;


                final Source source = sourceFactory.create(comp.getComponentName(), comp.getType());


                Configurables.configure(source, config);
                final Set<String> channelNames = config.getChannels();
                final List<Channel> channels = new ArrayList<Channel>();
                for (final String chName : channelNames) {
                    channels.add(conf.getChannels().get(chName));
                }


                final ChannelSelectorConfiguration selectorConfig = config.getSelectorConfiguration();


                final ChannelSelector selector = ChannelSelectorFactory.create(channels, selectorConfig);


                final ChannelProcessor channelProcessor = new ChannelProcessor(selector);
                Configurables.configure(channelProcessor, config);


                source.setChannelProcessor(channelProcessor);
                conf.getSourceRunners().put(comp.getComponentName(), SourceRunner.forSource(source));
            }
        }
        final Map<String, Context> sourceContexts = agentConf.getSourceContext();


        for (final String src : sources) {
            final Context context = sourceContexts.get(src);
            if (context != null) {
                final Source source = sourceFactory.create(src,
                    context.getString(BasicConfigurationConstants.CONFIG_TYPE));
                final List<Channel> channels = new ArrayList<Channel>();
                Configurables.configure(source, context);
                final String[] channelNames =
                    context.getString(BasicConfigurationConstants.CONFIG_CHANNELS).split("\\s+");
                for (final String chName : channelNames) {
                    channels.add(conf.getChannels().get(chName));
                }


                final Map<String, String> selectorConfig = context.getSubProperties(
                    BasicConfigurationConstants.CONFIG_SOURCE_CHANNELSELECTOR_PREFIX);


                final ChannelSelector selector = ChannelSelectorFactory.create(channels, selectorConfig);


                final ChannelProcessor channelProcessor = new ChannelProcessor(selector);
                Configurables.configure(channelProcessor, context);


                source.setChannelProcessor(channelProcessor);
                conf.getSourceRunners().put(src, SourceRunner.forSource(source));
            }
        }
    }

View Full Code Here

        for (String sourceName : sources) {
            ComponentConfiguration comp = compMap.get(sourceName);
            if (comp != null) {
                SourceConfiguration config = (SourceConfiguration) comp;


                Source source = sourceFactory.create(comp.getComponentName(), comp.getType());


                Configurables.configure(source, config);
                Set<String> channelNames = config.getChannels();
                List<Channel> channels = new ArrayList<Channel>();
                for (String chName : channelNames) {
                    channels.add(conf.getChannels().get(chName));
                }


                ChannelSelectorConfiguration selectorConfig = config.getSelectorConfiguration();


                ChannelSelector selector = ChannelSelectorFactory.create(channels, selectorConfig);


                ChannelProcessor channelProcessor = new ChannelProcessor(selector);
                Configurables.configure(channelProcessor, config);


                source.setChannelProcessor(channelProcessor);
                conf.getSourceRunners().put(comp.getComponentName(), SourceRunner.forSource(source));
            }
        }
        Map<String, Context> sourceContexts = agentConf.getSourceContext();


        for (String src : sources) {
            Context context = sourceContexts.get(src);
            if (context != null){
                Source source = sourceFactory.create(src, context.getString(BasicConfigurationConstants.CONFIG_TYPE));
                List<Channel> channels = new ArrayList<Channel>();
                Configurables.configure(source, context);
                String[] channelNames = context.getString(BasicConfigurationConstants.CONFIG_CHANNELS).split("\\s+");
                for (String chName : channelNames) {
                    channels.add(conf.getChannels().get(chName));
                }


                Map<String, String> selectorConfig = context.getSubProperties(
                    BasicConfigurationConstants.CONFIG_SOURCE_CHANNELSELECTOR_PREFIX);


                ChannelSelector selector = ChannelSelectorFactory.create(channels, selectorConfig);


                ChannelProcessor channelProcessor = new ChannelProcessor(selector);
                Configurables.configure(channelProcessor, context);


                source.setChannelProcessor(channelProcessor);
                conf.getSourceRunners().put(src, SourceRunner.forSource(source));
            }
        }
    }

View Full Code Here

        for (final String sourceName : sources) {
            final ComponentConfiguration comp = compMap.get(sourceName);
            if (comp != null) {
                final SourceConfiguration config = (SourceConfiguration) comp;


                final Source source = sourceFactory.create(comp.getComponentName(), comp.getType());


                Configurables.configure(source, config);
                final Set<String> channelNames = config.getChannels();
                final List<Channel> channels = new ArrayList<Channel>();
                for (final String chName : channelNames) {
                    channels.add(conf.getChannels().get(chName));
                }


                final ChannelSelectorConfiguration selectorConfig = config.getSelectorConfiguration();


                final ChannelSelector selector = ChannelSelectorFactory.create(channels, selectorConfig);


                final ChannelProcessor channelProcessor = new ChannelProcessor(selector);
                Configurables.configure(channelProcessor, config);


                source.setChannelProcessor(channelProcessor);
                conf.getSourceRunners().put(comp.getComponentName(), SourceRunner.forSource(source));
            }
        }
        final Map<String, Context> sourceContexts = agentConf.getSourceContext();


        for (final String src : sources) {
            final Context context = sourceContexts.get(src);
            if (context != null) {
                final Source source = sourceFactory.create(src,
                    context.getString(BasicConfigurationConstants.CONFIG_TYPE));
                final List<Channel> channels = new ArrayList<Channel>();
                Configurables.configure(source, context);
                final String[] channelNames =
                    context.getString(BasicConfigurationConstants.CONFIG_CHANNELS).split("\\s+");
                for (final String chName : channelNames) {
                    channels.add(conf.getChannels().get(chName));
                }


                final Map<String, String> selectorConfig = context.getSubProperties(
                    BasicConfigurationConstants.CONFIG_SOURCE_CHANNELSELECTOR_PREFIX);


                final ChannelSelector selector = ChannelSelectorFactory.create(channels, selectorConfig);


                final ChannelProcessor channelProcessor = new ChannelProcessor(selector);
                Configurables.configure(channelProcessor, context);


                source.setChannelProcessor(channelProcessor);
                conf.getSourceRunners().put(src, SourceRunner.forSource(source));
            }
        }
    }

View Full Code Here

        for (final String sourceName : sources) {
            final ComponentConfiguration comp = compMap.get(sourceName);
            if (comp != null) {
                final SourceConfiguration config = (SourceConfiguration) comp;


                final Source source = sourceFactory.create(comp.getComponentName(), comp.getType());


                Configurables.configure(source, config);
                final Set<String> channelNames = config.getChannels();
                final List<Channel> channels = new ArrayList<Channel>();
                for (final String chName : channelNames) {
                    channels.add(conf.getChannels().get(chName));
                }


                final ChannelSelectorConfiguration selectorConfig = config.getSelectorConfiguration();


                final ChannelSelector selector = ChannelSelectorFactory.create(channels, selectorConfig);


                final ChannelProcessor channelProcessor = new ChannelProcessor(selector);
                Configurables.configure(channelProcessor, config);


                source.setChannelProcessor(channelProcessor);
                conf.getSourceRunners().put(comp.getComponentName(), SourceRunner.forSource(source));
            }
        }
        final Map<String, Context> sourceContexts = agentConf.getSourceContext();


        for (final String src : sources) {
            final Context context = sourceContexts.get(src);
            if (context != null) {
                final Source source = sourceFactory.create(src,
                    context.getString(BasicConfigurationConstants.CONFIG_TYPE));
                final List<Channel> channels = new ArrayList<Channel>();
                Configurables.configure(source, context);
                final String[] channelNames =
                    context.getString(BasicConfigurationConstants.CONFIG_CHANNELS).split("\\s+");
                for (final String chName : channelNames) {
                    channels.add(conf.getChannels().get(chName));
                }


                final Map<String, String> selectorConfig = context.getSubProperties(
                    BasicConfigurationConstants.CONFIG_SOURCE_CHANNELSELECTOR_PREFIX);


                final ChannelSelector selector = ChannelSelectorFactory.create(channels, selectorConfig);


                final ChannelProcessor channelProcessor = new ChannelProcessor(selector);
                Configurables.configure(channelProcessor, context);


                source.setChannelProcessor(channelProcessor);
                conf.getSourceRunners().put(src, SourceRunner.forSource(source));
            }
        }
    }

View Full Code Here

0 1 2 3

TOP

Related Classes of org.apache.flume.Source

org.apache.flume.agent.embedded.EmbeddedAgent

org.apache.flume.conf.file.JsonFileConfigurationProvider

org.apache.flume.conf.properties.PropertiesFileConfigurationProvider

org.apache.flume.node.AbstractConfigurationProvider

org.apache.flume.node.TestAbstractLogicalNodeManager

org.apache.flume.source.DefaultSourceFactory

org.apache.flume.source.EventDrivenSourceRunner

org.apache.flume.source.jms.TestJMSSourceCreation

org.apache.flume.source.PollableSourceRunner

org.apache.flume.source.TestDefaultSourceFactory

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.