Examples of TopologyBuilder


Examples of backtype.storm.topology.TopologyBuilder

        HBaseLookupBolt hBaseLookupBolt = new HBaseLookupBolt("WordCount", mapper, rowToTupleMapper)
                .withConfigKey("hbase.conf")
                .withProjectionCriteria(projectionCriteria);

        //wordspout -> lookupbolt -> totalCountBolt
        TopologyBuilder builder = new TopologyBuilder();
        builder.setSpout(WORD_SPOUT, spout, 1);
        builder.setBolt(LOOKUP_BOLT, hBaseLookupBolt, 1).shuffleGrouping(WORD_SPOUT);
        builder.setBolt(TOTAL_COUNT_BOLT, totalBolt, 1).fieldsGrouping(LOOKUP_BOLT, new Fields("columnName"));

        if (args.length == 1) {
            LocalCluster cluster = new LocalCluster();
            cluster.submitTopology("test", config, builder.createTopology());
            Thread.sleep(30000);
            cluster.killTopology("test");
            cluster.shutdown();
            System.exit(0);
        } else if (args.length == 2) {
            StormSubmitter.submitTopology(args[1], config, builder.createTopology());
        } else{
            System.out.println("Usage: LookupWordCount <hbase.rootdir>");
        }
    }
View Full Code Here

Examples of backtype.storm.topology.TopologyBuilder

public class HBaseCountersTopology {
  /**
   * @param args
   */
  public static void main(String[] args) {
    TopologyBuilder builder = new TopologyBuilder();

    // Add test spout
    builder.setSpout("spout", new TestSpout(), 1);

    // Build TupleTableConifg
    TupleTableConfig config = new TupleTableConfig("shorturl", "shortid");
    config.setBatch(false);
    /*
     * By default the HBaseCountersBolt will use the tuple output fields value
     * to set the CQ name. For example if the 'date' output field exists in the
     * tuple then its value (e.g. "YYYYMMDD") will be used to set the counters
     * CQ. However, the 'clicks' output field does not exist in the tuple so in
     * this case the counters CQ will be set to the given field name 'clicks'.
     */
    config.addColumn("data", "clicks");
    config.addColumn("daily", "date");

    // Add HBaseBolt
    builder.setBolt("hbase-counters", new HBaseCountersBolt(config), 1)
        .shuffleGrouping("spout");

    Config stormConf = new Config();
    stormConf.setDebug(true);

    LocalCluster cluster = new LocalCluster();
    cluster
        .submitTopology("hbase-example", stormConf, builder.createTopology());

    Utils.sleep(10000);
    cluster.shutdown();
  }
View Full Code Here

Examples of backtype.storm.topology.TopologyBuilder

public class HBaseExampleTopology {
  /**
   * @param args
   */
  public static void main(String[] args) {
    TopologyBuilder builder = new TopologyBuilder();

    // Add test spout
    builder.setSpout("spout", new TestSpout(), 1);

    // Build TupleTableConifg
    TupleTableConfig config = new TupleTableConfig("shorturl", "shortid");
    config.setBatch(false);
    config.addColumn("data", "url");
    config.addColumn("data", "user");
    config.addColumn("data", "date");

    // Add HBaseBolt
    builder.setBolt("hbase", new HBaseBolt(config), 1).shuffleGrouping("spout");

    Config stormConf = new Config();
    stormConf.setDebug(true);

    LocalCluster cluster = new LocalCluster();
    cluster
        .submitTopology("hbase-example", stormConf, builder.createTopology());

    Utils.sleep(10000);
    cluster.shutdown();
  }
View Full Code Here

Examples of backtype.storm.topology.TopologyBuilder

      throw new RuntimeException("QueryPlan " + queryName + " doesn't exist in Main.java");
    return queryPlan;
  }

  private static TopologyBuilder createTopology(QueryPlan qp, Config conf) {
    final TopologyBuilder builder = new TopologyBuilder();
    final TopologyKiller killer = new TopologyKiller(builder);

    // DST_ORDERING is the optimized version, so it's used by default
    final int partitioningType = StormJoin.DST_ORDERING;
View Full Code Here

Examples of backtype.storm.topology.TopologyBuilder

  public Main(QueryPlan queryPlan, Map map, String confPath) {
    final Config conf = SystemParameters.mapToStormConfig(map);

    addVariablesToMap(conf, confPath);
    putBatchSizes(queryPlan, conf);
    final TopologyBuilder builder = createTopology(queryPlan, conf);
    StormWrapper.submitTopology(conf, builder);
  }
View Full Code Here

Examples of backtype.storm.topology.TopologyBuilder

    final Config conf = SystemParameters.fileToStormConfig(confPath);
    final QueryPlan queryPlan = chooseQueryPlan(conf);

    addVariablesToMap(conf, confPath);
    putBatchSizes(queryPlan, conf);
    final TopologyBuilder builder = createTopology(queryPlan, conf);
    StormWrapper.submitTopology(conf, builder);
  }
View Full Code Here

Examples of backtype.storm.topology.TopologyBuilder

        throw new LumifyException("You must specify one input method");
    }

    public StormTopology createTopology(int parallelismHint) {
        TopologyBuilder builder = new TopologyBuilder();
        createTweetProcessingTopology(builder, parallelismHint);
        return builder.createTopology();
    }
View Full Code Here

Examples of backtype.storm.topology.TopologyBuilder

    protected String getTopologyName() {
        return TOPOLOGY_NAME;
    }

    public StormTopology createTopology(int parallelismHint) {
        TopologyBuilder builder = new TopologyBuilder();
        createGraphPropertyTopology(builder, parallelismHint);
        return builder.createTopology();
    }
View Full Code Here

Examples of backtype.storm.topology.TopologyBuilder

    String assignmentports=String.valueOf(stormconf.get(MdrillDefaultTaskAssignment.MDRILL_ASSIGNMENT_PORTS+"."+topologyName));
    conf.put(CustomAssignment.TOPOLOGY_CUSTOM_ASSIGNMENT, MdrillDefaultTaskAssignment.class.getName());
    conf.put(MdrillDefaultTaskAssignment.MDRILL_ASSIGNMENT_DEFAULT, assignment);
    conf.put(MdrillDefaultTaskAssignment.MDRILL_ASSIGNMENT_PORTS, assignmentports);

    TopologyBuilder builder = new TopologyBuilder();
   
    if(args.length>=6)
    {
      String tttime=args[5];
      for(String prefix:prefixlist)
      {
        if(!tttime.equals("0"))
        {
          conf.put(prefix+"-start-time", tttime);
        }
      }
    }
   
    if(args.length>=7)
    {
      String[] prefix_timeist=args[6].split(";");
      for(String prefix_time:prefix_timeist)
      {
        String[] prefix_time_col=prefix_time.split(":");
        if(prefix_time_col.length>1)
        {
          conf.put(prefix_time_col[0]+"-start-time", prefix_time_col[1]);
        }
      }
    }
   
    for(String prefix:prefixlist)
    {
      conf.put(prefix+"-validate-time", System.currentTimeMillis());
    }
   
   
    for(String prefix:prefixlist)
    {
      String mode=String.valueOf(conf.get(prefix+"-mode"));
      String threadconfig=conf.get(prefix+"-threads")!=null?String.valueOf(conf.get(prefix+"-threads")):String.valueOf(workescount);
      int threads=workescount;
      try{
        threads=Integer.parseInt(threadconfig);
      }catch(Throwable e)
      {
        threads=workescount;
      }
      int threads_reduce=threads;
      String threadconfig_reduce=conf.get(prefix+"-threads_reduce")!=null?String.valueOf(conf.get(prefix+"-threads_reduce")):String.valueOf(threads_reduce);
      try{
        threads_reduce=Integer.parseInt(threadconfig_reduce);
      }catch(Throwable e)
      {
        threads_reduce=threads;
      }
     
      if(mode.equals("local"))
      {
        builder.setSpout("map_"+prefix, new ImportSpoutLocal(prefix), threads);
      }else{
        builder.setSpout("map_"+prefix, new ImportSpout(prefix), threads);
        builder.setBolt("reduce_"+prefix, new ImportBolt(prefix), threads_reduce).fieldsGrouping("map_"+prefix, new Fields("key") );
      }
    }
   
    StormSubmitter.submitTopology(topologyName, conf,builder.createTopology());

  }
View Full Code Here

Examples of backtype.storm.topology.TopologyBuilder

    conf.putAll(stormconf);
    conf.setMessageTimeoutSecs(10);
   
    String topologyName=args[0];
    String[] prefixlist=args[1].split(",");
    TopologyBuilder builder = new TopologyBuilder();
   
    if(args.length>=6)
    {
      String tttime=args[5];
      for(String prefix:prefixlist)
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.