Examples of TopologyBuilder

backtype.storm.topology.TopologyBuilder

TopologyBuilder exposes the Java API for specifying a topology for Storm to execute. Topologies are Thrift structures in the end, but since the Thrift API is so verbose, TopologyBuilder greatly eases the process of creating topologies. The template for creating and submitting a topology looks something like:

 TopologyBuilder builder = new TopologyBuilder(); builder.setSpout("1", new TestWordSpout(true), 5); builder.setSpout("2", new TestWordSpout(true), 3); builder.setBolt("3", new TestWordCounter(), 3) .fieldsGrouping("1", new Fields("word")) .fieldsGrouping("2", new Fields("word")); builder.setBolt("4", new TestGlobalCount()) .globalGrouping("1"); Map conf = new HashMap(); conf.put(Config.TOPOLOGY_WORKERS, 4); StormSubmitter.submitTopology("mytopology", conf, builder.createTopology());

Running the exact same topology in local mode (in process), and configuring it to log all tuples emitted, looks like the following. Note that it lets the topology run for 10 seconds before shutting down the local cluster.

 TopologyBuilder builder = new TopologyBuilder(); builder.setSpout("1", new TestWordSpout(true), 5); builder.setSpout("2", new TestWordSpout(true), 3); builder.setBolt("3", new TestWordCounter(), 3) .fieldsGrouping("1", new Fields("word")) .fieldsGrouping("2", new Fields("word")); builder.setBolt("4", new TestGlobalCount()) .globalGrouping("1"); Map conf = new HashMap(); conf.put(Config.TOPOLOGY_WORKERS, 4); conf.put(Config.TOPOLOGY_DEBUG, true); LocalCluster cluster = new LocalCluster(); cluster.submitTopology("mytopology", conf, builder.createTopology()); Utils.sleep(10000); cluster.shutdown();

The pattern for TopologyBuilder is to map component ids to components using the setSpout and setBolt methods. Those methods return objects that are then used to declare the inputs for that component.

org.apache.hadoop.gateway.topology.builder.TopologyBuilder
org.apache.stratos.cloud.controller.consumers.TopologyBuilder
This class is responsible for generating the service topology and publishing it to a topic.

Examples of backtype.storm.topology.TopologyBuilder

  private final String topologyName;
  private final Config topologyConfig;
  private final int runtimeInSeconds;


  public RollingTopWords() throws InterruptedException {
    builder = new TopologyBuilder();
    topologyName = "slidingWindowCounts";
    topologyConfig = createTopologyConfiguration();
    runtimeInSeconds = DEFAULT_RUNTIME_IN_SECONDS;


    wireTopology();

View Full Code Here

Examples of backtype.storm.topology.TopologyBuilder

        String accessToken = args[2]; 
        String accessTokenSecret = args[3];
        String[] arguments = args.clone();
        String[] keyWords = Arrays.copyOfRange(arguments, 4, arguments.length);
        
        TopologyBuilder builder = new TopologyBuilder();
        
        builder.setSpout("spoutId", new TwitterSampleSpout(consumerKey, consumerSecret,
                                accessToken, accessTokenSecret, keyWords));
        builder.setBolt("print", new PrinterBolt())
                .shuffleGrouping("spout");
                
                
        Config conf = new Config();
        
        
        LocalCluster cluster = new LocalCluster();
        
        cluster.submitTopology("test", conf, builder.createTopology());
        
        Utils.sleep(10000);
        cluster.shutdown();
    }

View Full Code Here

Examples of backtype.storm.topology.TopologyBuilder

        ConfigurableTopology.start(new CrawlTopology(), args);
    }


    @Override
    protected int run(String[] args) {
        TopologyBuilder builder = new TopologyBuilder();


        builder.setSpout("spout", new RandomURLSpout());


        builder.setBolt("partitioner", new URLPartitionerBolt())
                .shuffleGrouping("spout");


        builder.setBolt("fetch", new FetcherBolt()).fieldsGrouping(
                "partitioner", new Fields("key"));


        builder.setBolt("parse", new ParserBolt()).shuffleGrouping("fetch");


        builder.setBolt("index", new IndexerBolt()).shuffleGrouping("parse");


        conf.registerMetricsConsumer(DebugMetricConsumer.class);


        return submit("crawl", conf, builder);
    }

View Full Code Here

Examples of backtype.storm.topology.TopologyBuilder

    }
  }


  public static void main(String[] args) throws Exception {


    TopologyBuilder builder = new TopologyBuilder();


    builder.setSpout("spout", new RandomSentenceSpout(), 5);


    builder.setBolt("split", new SplitSentence(), 8).shuffleGrouping("spout");
    builder.setBolt("count", new WordCount(), 12).fieldsGrouping("split", new Fields("word"));


    Config conf = new Config();
    conf.setDebug(true);




    if (args != null && args.length > 0) {
      conf.setNumWorkers(3);


      StormSubmitter.submitTopology(args[0], conf, builder.createTopology());
    }
    else {
      conf.setMaxTaskParallelism(3);


      LocalCluster cluster = new LocalCluster();
      cluster.submitTopology("word-count", conf, builder.createTopology());


      Thread.sleep(10000);


      cluster.shutdown();
    }

View Full Code Here

Examples of backtype.storm.topology.TopologyBuilder

    }


  }


  public static void main(String[] args) {
    TopologyBuilder builder = new TopologyBuilder();
    LocalDRPC drpc = new LocalDRPC();


    DRPCSpout spout = new DRPCSpout("exclamation", drpc);
    builder.setSpout("drpc", spout);
    builder.setBolt("exclaim", new ExclamationBolt(), 3).shuffleGrouping("drpc");
    builder.setBolt("return", new ReturnResults(), 3).shuffleGrouping("exclaim");


    LocalCluster cluster = new LocalCluster();
    Config conf = new Config();
    cluster.submitTopology("exclaim", conf, builder.createTopology());


    System.out.println(drpc.execute("exclamation", "aaa"));
    System.out.println(drpc.execute("exclamation", "bbb"));


  }

View Full Code Here

Examples of org.apache.hadoop.gateway.topology.builder.TopologyBuilder

    Digester digester = loader.newDigester();
    String name = "org/apache/hadoop/gateway/topology/xml/simple-topology-knox-format.xml";
    URL url = ClassLoader.getSystemResource( name );
    assertThat( "Failed to find URL for resource " + name, url, notNullValue() );
    File file = new File( url.getFile() );
    TopologyBuilder topologyBuilder = digester.parse( url );
    Topology topology = topologyBuilder.build();
    assertThat( "Failed to parse resource " + name, topology, notNullValue() );
    topology.setTimestamp( file.lastModified() );


    assertThat( topology.getName(), is( "topology" ) );
    assertThat( topology.getTimestamp(), is( file.lastModified() ) );

View Full Code Here

Examples of org.apache.hadoop.gateway.topology.builder.TopologyBuilder

    Digester digester = loader.newDigester();
    String name = "org/apache/hadoop/gateway/topology/xml/service-param-topology-knox-format.xml";
    URL url = ClassLoader.getSystemResource( name );
    assertThat( "Failed to find URL for resource " + name, url, notNullValue() );
    File file = new File( url.getFile() );
    TopologyBuilder topologyBuilder = digester.parse( url );
    Topology topology = topologyBuilder.build();
    assertThat( "Failed to parse resource " + name, topology, notNullValue() );
    topology.setTimestamp( file.lastModified() );


    assertThat( topology.getName(), is( "test-topology-name" ) );
    assertThat( topology.getTimestamp(), is( file.lastModified() ) );

View Full Code Here

Examples of org.apache.hadoop.gateway.topology.builder.TopologyBuilder

    Digester digester = loader.newDigester();
    String name = "org/apache/hadoop/gateway/topology/xml/simple-topology-ambari-format.conf";
    URL url = ClassLoader.getSystemResource( name );
    assertThat( "Failed to find URL for resource " + name, url, notNullValue() );
    File file = new File( url.getFile() );
    TopologyBuilder topologyBuilder = digester.parse( url );
    Topology topology = topologyBuilder.build();
    assertThat( "Failed to parse resource " + name, topology, notNullValue() );
    topology.setTimestamp( file.lastModified() );


    assertThat( topology.getName(), is( "topology2" ) );
    assertThat( topology.getTimestamp(), is( file.lastModified() ) );

View Full Code Here

Examples of org.apache.hadoop.gateway.topology.builder.TopologyBuilder

    Digester digester = loader.newDigester();
    String name = "org/apache/hadoop/gateway/topology/xml/service-param-topology-ambari-format.conf";
    URL url = ClassLoader.getSystemResource( name );
    assertThat( "Failed to find URL for resource " + name, url, notNullValue() );
    File file = new File( url.getFile() );
    TopologyBuilder topologyBuilder = digester.parse( url );
    Topology topology = topologyBuilder.build();
    assertThat( "Failed to parse resource " + name, topology, notNullValue() );
    topology.setTimestamp( file.lastModified() );


    assertThat( topology.getName(), is( "test-topology-name" ) );
    assertThat( topology.getTimestamp(), is( file.lastModified() ) );

View Full Code Here

Examples of org.apache.hadoop.gateway.topology.builder.TopologyBuilder

    return topology;
  }


  private static Topology loadTopologyAttempt( File file ) throws IOException, SAXException, URISyntaxException {
    Topology topology;Digester digester = digesterLoader.newDigester();
    TopologyBuilder topologyBuilder = digester.parse( FileUtils.openInputStream( file ) );
    topology = topologyBuilder.build();
    topology.setUri( file.toURI() );
    topology.setName( FilenameUtils.removeExtension( file.getName() ) );
    topology.setTimestamp( file.lastModified() );
    return topology;
  }

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.