Package storm.trident

Examples of storm.trident.TridentTopology

94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
    TridentConfig config = new TridentConfig("shorturl", "shortid");
    config.setBatch(false);

    StateFactory state = HBaseAggregateState.transactional(config);

    TridentTopology topology = new TridentTopology();
    topology
        .newStream("spout", spout)
        .each(new Fields("shortid", "date"), new DatePartitionFunction(),
            new Fields("cf", "cq")).project(new Fields("shortid", "cf", "cq"))
        .groupBy(new Fields("shortid", "cf", "cq"))
        .persistentAggregate(state, new Count(), new Fields("count"));

    Config conf = new Config();
    LocalCluster cluster = new LocalCluster();
    cluster.submitTopology("hbase-trident-aggregate", conf, topology.build());

    Utils.sleep(5000);
    cluster.shutdown();
  }
View Full Code Here
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
    updateConfig.setBatch(false);
    updateConfig.addColumn("data", "url");
    updateConfig.addColumn("data", "user");
    updateConfig.addColumn("data", "date");

    TridentTopology topology = new TridentTopology();
    topology.newStream("shorturls", spout).partitionPersist(
        new HBaseValueFactory(updateConfig),
        new Fields("shortid", "url", "user", "date"), new HBaseValueUpdater());

    Config conf = new Config();
    LocalCluster cluster = new LocalCluster();

    cluster.submitTopology("hbase-trident-updater", conf, topology.build());
    Utils.sleep(10000);
    cluster.shutdown();
  }
View Full Code Here
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
                new Values("four score and seven years ago"),
                new Values("how many apples can you eat"),
                new Values("to be or not to be the person"));
        spout.setCycle(false);
       
        TridentTopology topology = new TridentTopology();
        TridentState wordCounts =
              topology.newStream("spout1", spout)
                .parallelismHint(16)
                .each(new Fields("sentence"), new Split(), new Fields("word"))
                .groupBy(new Fields("word"))
                .persistentAggregate(new MemoryMapState.Factory(),
                                     new Count(), new Fields("count"))
                .parallelismHint(16);
               
        topology.newDRPCStream("words", drpc)
                .each(new Fields("args"), new Split(), new Fields("word"))
                .groupBy(new Fields("word"))
                .stateQuery(wordCounts, new Fields("word"), new MapGet(), new Fields("count"))
                .each(new Fields("count"), new FilterNull())
                //.aggregate(new Fields("count"), new Sum(), new Fields("sum"))
                ;
       
        return topology.build();
    }
View Full Code Here
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
    }
   
    public static StormTopology buildTopology(LocalDRPC drpc) {
        _spout = new FeederSpout(new Fields("sentence"));
       
        TridentTopology topology = new TridentTopology();
        TridentState wordCounts =
              topology.newStream("spout1", _spout)
                .parallelismHint(16)
                .each(new Fields("sentence"), new Split(), new Fields("word"))
                .groupBy(new Fields("word"))
                .persistentAggregate(new MemoryMapState.Factory(),
                                     new Count(), new Fields("count"))
                .parallelismHint(16);
       
        topology.newDRPCStream("words", drpc)
                .each(new Fields("args"), new Split(), new Fields("word"))
                .groupBy(new Fields("word"))
                .stateQuery(wordCounts, new Fields("word"), new MapGet(), new Fields("count"))
                .each(new Fields("count"), new FilterNull())
                //.aggregate(new Fields("count"), new Sum(), new Fields("sum"))
                ;
       
        return topology.build();
    }
View Full Code Here
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
        }
       
    }
   
    public static StormTopology buildTopology(LocalDRPC drpc) {
        TridentTopology topology = new TridentTopology();
        TridentState urlToTweeters =
                topology.newStaticState(
                    new StaticSingleKeyMapState.Factory(TWEETERS_DB));
        TridentState tweetersToFollowers =
                topology.newStaticState(
                    new StaticSingleKeyMapState.Factory(FOLLOWERS_DB));
       
       
        topology.newDRPCStream("reach", drpc)
                .stateQuery(urlToTweeters, new Fields("args"), new MapGet(), new Fields("tweeters"))
                .each(new Fields("tweeters"), new ExpandList(), new Fields("tweeter"))
                .shuffle()
                .stateQuery(tweetersToFollowers, new Fields("tweeter"), new MapGet(), new Fields("followers"))
                .each(new Fields("followers"), new ExpandList(), new Fields("follower"))
                .groupBy(new Fields("follower"))
                .aggregate(new One(), new Fields("one"))
                .aggregate(new Fields("one"), new Sum(), new Fields("reach"));
        return topology.build();
    }
View Full Code Here
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
        FixedBatchSpout spout = new FixedBatchSpout(new Fields("sentence", "key"), 1000, new Values("the cow jumped over the moon", 1l),
                new Values("the man went to the store and bought some candy", 2l), new Values("four score and seven years ago", 3l),
                new Values("how many apples can you eat", 4l), new Values("to be or not to be the person", 5l));
        spout.setCycle(true);

        TridentTopology topology = new TridentTopology();
        Stream stream = topology.newStream("spout1", spout);

        Fields hdfsFields = new Fields("sentence", "key");

        FileNameFormat fileNameFormat = new DefaultFileNameFormat()
                .withPath("/trident")
                .withPrefix("trident")
                .withExtension(".txt");

        RecordFormat recordFormat = new DelimitedRecordFormat()
                .withFields(hdfsFields);

        FileRotationPolicy rotationPolicy = new FileSizeRotationPolicy(5.0f, FileSizeRotationPolicy.Units.MB);

        HdfsState.Options options = new HdfsState.HdfsFileOptions()
                .withFileNameFormat(fileNameFormat)
                .withRecordFormat(recordFormat)
                .withRotationPolicy(rotationPolicy)
                .withFsUrl(hdfsUrl);

        StateFactory factory = new HdfsStateFactory().withOptions(options);

        TridentState state = stream
                .partitionPersist(factory, hdfsFields, new HdfsUpdater(), new Fields());

        return topology.build();
    }
View Full Code Here
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
        FixedBatchSpout spout = new FixedBatchSpout(new Fields("sentence", "key"), 1000, new Values("the cow jumped over the moon", 1l),
                new Values("the man went to the store and bought some candy", 2l), new Values("four score and seven years ago", 3l),
                new Values("how many apples can you eat", 4l), new Values("to be or not to be the person", 5l));
        spout.setCycle(true);

        TridentTopology topology = new TridentTopology();
        Stream stream = topology.newStream("spout1", spout);

        Fields hdfsFields = new Fields("sentence", "key");

        FileNameFormat fileNameFormat = new DefaultFileNameFormat()
                .withPath("/trident")
                .withPrefix("trident")
                .withExtension(".seq");

        FileRotationPolicy rotationPolicy = new FileSizeRotationPolicy(5.0f, FileSizeRotationPolicy.Units.MB);

        HdfsState.Options seqOpts = new HdfsState.SequenceFileOptions()
                .withFileNameFormat(fileNameFormat)
                .withSequenceFormat(new DefaultSequenceFormat("key", "sentence"))
                .withRotationPolicy(rotationPolicy)
                .withFsUrl(hdfsUrl)
                .addRotationAction(new MoveFileAction().toDestination("/dest2/"));

        StateFactory factory = new HdfsStateFactory().withOptions(seqOpts);

        TridentState state = stream
                .partitionPersist(factory, hdfsFields, new HdfsUpdater(), new Fields());

        return topology.build();
    }
View Full Code Here
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
                new Values("needs", "1"),
                new Values("javadoc", "1")
        );
        spout.setCycle(true);

        TridentTopology topology = new TridentTopology();
        Stream stream = topology.newStream("spout1", spout);

        TridentKafkaStateFactory stateFactory = new TridentKafkaStateFactory()
                .withKafkaTopicSelector(new DefaultTopicSelector("test"))
                .withTridentTupleToKafkaMapper(new FieldNameBasedTupleToKafkaMapper("word", "count"));
        stream.partitionPersist(stateFactory, fields, new TridentKafkaUpdater(), new Fields());

        return topology.build();
    }
View Full Code Here
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
                new Values("the man went to the store and bought some candy"),
                new Values("four score and seven years ago"),
                new Values("how many apples can you eat"));
        spout.setCycle(false);

        TridentTopology topology = new TridentTopology();

        HashMap<String, Object> clientConfig = new HashMap<String, Object>();
        clientConfig.put(StormCassandraConstants.CASSANDRA_HOST, "localhost:9160");
        clientConfig.put(StormCassandraConstants.CASSANDRA_STATE_KEYSPACE, KEYSPACE);
        Config config = new Config();
        config.setMaxSpoutPending(25);
        config.put("cassandra.config", clientConfig);

        StateFactory cassandraStateFactory = null;
        Options options = null;
        switch(txType){
        case TRANSACTIONAL:
            options = new Options<TransactionalValue>();
            options.columnFamily = "transactional";
            cassandraStateFactory = CassandraMapState.transactional(options);           
            break;
           
        case OPAQUE:
            options = new Options<OpaqueValue>();
            options.columnFamily = "opaque";
            cassandraStateFactory = CassandraMapState.opaque(options);
            break;
           
        case NON_TRANSACTIONAL:
            options = new Options<Object>();
            options.columnFamily = "nontransactional";
            cassandraStateFactory = CassandraMapState.nonTransactional(options);
            break;
        }
       

        TridentState wordCounts = topology.newStream("spout1", spout)
                .each(new Fields("sentence"), new Split(), new Fields("word")).groupBy(new Fields("word"))
                .persistentAggregate(cassandraStateFactory, new Count(), new Fields("count")).parallelismHint(1);

        LocalDRPC client = new LocalDRPC();
        topology.newDRPCStream("words", client).each(new Fields("args"), new Split(), new Fields("word"))
                .groupBy(new Fields("word"))
                .stateQuery(wordCounts, new Fields("word"), new MapGet(), new Fields("count"))
                .each(new Fields("count"), new FilterNull())
                .aggregate(new Fields("count"), new Sum(), new Fields("sum"));

        LocalCluster cluster = new LocalCluster();
        cluster.submitTopology("test", config, topology.build());

            Thread.sleep(10000);


            assertEquals("[[5]]", client.execute("words", "cat dog the man")); // 5
View Full Code Here
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
    FixedBatchSpout spout = new FixedBatchSpout(new Fields("sentence"), 3, new Values("the cow jumped over the moon"),
        new Values("the man went to the store and bought some candy"), new Values("four score and seven years ago"),
        new Values("how many apples can you eat"), new Values("to be or not to be the person"));
    spout.setCycle(true);

    TridentTopology topology = new TridentTopology();
    TridentState wordCounts = topology.newStream("spout1", spout).parallelismHint(16).each(new Fields("sentence"),
        new Split(), new Fields("word")).groupBy(new Fields("word")).persistentAggregate(new MemoryMapState.Factory(),
        new Count(), new Fields("count")).parallelismHint(16);

    topology.newDRPCStream("words", drpc).each(new Fields("args"), new Split(), new Fields("word")).groupBy(new Fields(
        "word")).stateQuery(wordCounts, new Fields("word"), new MapGet(), new Fields("count")).each(new Fields("count"),
        new FilterNull()).aggregate(new Fields("count"), new Sum(), new Fields("sum"));
    return topology.build();
  }
View Full Code Here
TOP

Related Classes of storm.trident.TridentTopology

  • backtype.storm.contrib.hbase.examples.HBaseTridentAggregateTopology
  • backtype.storm.contrib.hbase.examples.HBaseTridentValueTopology
  • backtype.storm.drpc.DRPCSpout
  • backtype.storm.generated.GlobalStreamId
  • backtype.storm.generated.Grouping
  • backtype.storm.grouping.CustomStreamGrouping
  • backtype.storm.topology.BoltDeclarer
  • backtype.storm.tuple.Fields
  • com.datasalt.trident.Demo
  • com.datasalt.trident.examples.ParallelismExample1
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.