Examples of GlobalStreamId

backtype.storm.generated.GlobalStreamId

Examples of backtype.storm.generated.GlobalStreamId

      // user custom grouping by JavaObject
      JavaObject jobj = thrift_grouping.get_custom_object();
      CustomStreamGrouping g = Thrift.instantiateJavaObject(jobj);
      int myTaskId = topology_context.getThisTaskId();
      String componentId = topology_context.getComponentId(myTaskId);
      GlobalStreamId stream = new GlobalStreamId(componentId, streamId);
      custom_grouper = new MkCustomGrouper(topology_context, g, stream,
          out_tasks, myTaskId);
      grouperType = GrouperType.custom_obj;
    } else if (Grouping._Fields.CUSTOM_SERIALIZED.equals(fields)) {
      // user custom group by serialized Object
      byte[] obj = thrift_grouping.get_custom_serialized();
      CustomStreamGrouping g = (CustomStreamGrouping) Utils
          .deserialize(obj);
      int myTaskId = topology_context.getThisTaskId();
      String componentId = topology_context.getComponentId(myTaskId);
      GlobalStreamId stream = new GlobalStreamId(componentId, streamId);
      custom_grouper = new MkCustomGrouper(topology_context, g, stream,
          out_tasks, myTaskId);
      grouperType = GrouperType.custom_serialized;
    } else if (Grouping._Fields.DIRECT.equals(fields)) {
      // directly send to a special task

View Full Code Here

Examples of backtype.storm.generated.GlobalStreamId

    Map<GlobalStreamId, Double> processSum = JStormUtils
        .mergeMapList(processList);
    
    Map<GlobalStreamId, Double> process = new HashMap<GlobalStreamId, Double>();
    for (Entry<GlobalStreamId, Double> entry : processSum.entrySet()) {
        GlobalStreamId streamId = entry.getKey();
        Double value = entry.getValue();
        
        if (value != null && processList.size() > 0) {
            process.put(streamId, value/processList.size());
        }
    }


    for (Entry<String, Long> emittedEntry : emitted.entrySet()) {
      String outputStreamId = emittedEntry.getKey();
      Long emittedValue = emittedEntry.getValue();
      Double sendTpsValue = sendTps.get(outputStreamId);


      GlobalStreamId streamId = null;
      for (Entry<GlobalStreamId, Long> entry : acked.entrySet()) {
        String stream = entry.getKey().get_streamId();
        if (outputStreamId.equals(stream)) {
          streamId = entry.getKey();
          break;

View Full Code Here

Examples of backtype.storm.generated.GlobalStreamId

    }


    private BoltDeclarer grouping(String componentId, String streamId,
        Grouping grouping) {
      _commons.get(_boltId).put_to_inputs(
          new GlobalStreamId(componentId, streamId), grouping);
      return this;
    }

View Full Code Here

Examples of backtype.storm.generated.GlobalStreamId

    Map<GlobalStreamId, Grouping> spout_inputs = new HashMap<GlobalStreamId, Grouping>();
    Map<String, SpoutSpec> spout_ids = topology.get_spouts();
    for (Entry<String, SpoutSpec> spout : spout_ids.entrySet()) {
      String id = spout.getKey();


      GlobalStreamId stream = new GlobalStreamId(id, ACKER_INIT_STREAM_ID);


      Grouping group = Thrift.mkFieldsGrouping(JStormUtils.mk_list("id"));


      spout_inputs.put(stream, group);
    }


    Map<String, Bolt> bolt_ids = topology.get_bolts();
    Map<GlobalStreamId, Grouping> bolt_inputs = new HashMap<GlobalStreamId, Grouping>();
    for (Entry<String, Bolt> bolt : bolt_ids.entrySet()) {
      String id = bolt.getKey();


      GlobalStreamId streamAck = new GlobalStreamId(id,
          ACKER_ACK_STREAM_ID);
      Grouping groupAck = Thrift.mkFieldsGrouping(JStormUtils
          .mk_list("id"));


      GlobalStreamId streamFail = new GlobalStreamId(id,
          ACKER_FAIL_STREAM_ID);
      Grouping groupFail = Thrift.mkFieldsGrouping(JStormUtils
          .mk_list("id"));


      bolt_inputs.put(streamAck, groupAck);

View Full Code Here

Examples of backtype.storm.generated.GlobalStreamId

      List<String> initList = JStormUtils.mk_list("id", "init-val",
          "spout-task");
      common.put_to_streams(ACKER_INIT_STREAM_ID,
          Thrift.outputFields(initList));


      GlobalStreamId ack_ack = new GlobalStreamId(ACKER_COMPONENT_ID,
          ACKER_ACK_STREAM_ID);
      common.put_to_inputs(ack_ack, Thrift.mkDirectGrouping());


      GlobalStreamId ack_fail = new GlobalStreamId(ACKER_COMPONENT_ID,
          ACKER_FAIL_STREAM_ID);
      common.put_to_inputs(ack_fail, Thrift.mkDirectGrouping());
    }


    ret.put_to_bolts(ACKER_COMPONENT_ID, acker_bolt);

View Full Code Here

Examples of backtype.storm.generated.GlobalStreamId

    Map<GlobalStreamId, Double> processSum = JStormUtils
                .mergeMapList(processList);
        
        Map<GlobalStreamId, Double> process = new HashMap<GlobalStreamId, Double>();
        for (Entry<GlobalStreamId, Double> entry : processSum.entrySet()) {
            GlobalStreamId streamId = entry.getKey();
            Double value = entry.getValue();
            
            if (value != null && processList.size() > 0) {
                process.put(streamId, value/processList.size());
            }
        }


    for (Entry<String, Long> emittedEntry : emitted.entrySet()) {
      String outputStreamId = emittedEntry.getKey();
      Long emittedValue = emittedEntry.getValue();
      Double sendTpsValue = sendTps.get(outputStreamId);


      ComponentOutput co = new ComponentOutput();


      co.setValues(outputStreamId, emittedValue, sendTpsValue);


      coos.add(co);
    }


    for (Entry<GlobalStreamId, Double> entry : process.entrySet()) {
      GlobalStreamId streamId = entry.getKey();
      Double processValue = entry.getValue();


      Double recvTpsValue = recvTps.get(streamId);
      Long ackedValue = acked.get(streamId);
      Long failedValue = failed.get(streamId);

View Full Code Here

Examples of backtype.storm.generated.GlobalStreamId

        String stream) {
      return new IdStreamSpec(component, stream);
    }


    protected IdStreamSpec(String component, String stream) {
      _id = new GlobalStreamId(component, stream);
    }

View Full Code Here

Examples of backtype.storm.generated.GlobalStreamId

                    if(parent instanceof SpoutNode) {
                        componentId = spoutIds.get(parent);
                    } else {
                        componentId = boltIds.get(grouper.nodeGroup(parent));
                    }
                    d.grouping(new GlobalStreamId(componentId, n.streamId), n.thriftGrouping);
                } 
            }
        }
        
        return builder.buildTopology();

View Full Code Here

Examples of backtype.storm.generated.GlobalStreamId

    
    Map<GlobalStreamId, String> fleshOutStreamBatchIds(boolean includeCommitStream) {
        Map<GlobalStreamId, String> ret = new HashMap<GlobalStreamId, String>(_batchIds);
        Set<String> allBatches = new HashSet(_batchIds.values());
        for(String b: allBatches) {
            ret.put(new GlobalStreamId(masterCoordinator(b), MasterBatchCoordinator.BATCH_STREAM_ID), b);
            if(includeCommitStream) {
                ret.put(new GlobalStreamId(masterCoordinator(b), MasterBatchCoordinator.COMMIT_STREAM_ID), b);
            }
            // DO NOT include the success stream as part of the batch. it should not trigger coordination tuples,
            // and is just a metadata tuple to assist in cleanup, should not trigger batch tracking
        }
        
        for(String id: _spouts.keySet()) {
            TransactionalSpoutComponent c = _spouts.get(id);
            if(c.batchGroupId!=null) {
                ret.put(new GlobalStreamId(spoutCoordinator(id), MasterBatchCoordinator.BATCH_STREAM_ID), c.batchGroupId);
            }
        }


        //this takes care of setting up coord streams for spouts and bolts
        for(GlobalStreamId s: _batchIds.keySet()) {
            String b = _batchIds.get(s);
            ret.put(new GlobalStreamId(s.get_componentId(), TridentBoltExecutor.COORD_STREAM(b)), b);
        }
        
        return ret;
    }

View Full Code Here

Examples of backtype.storm.generated.GlobalStreamId

                }
                spec.coords.put(s.get_componentId(), ct);
            }
            
            for(String b: c.committerBatches) {
                specs.get(b).commitStream = new GlobalStreamId(masterCoordinator(b), MasterBatchCoordinator.COMMIT_STREAM_ID);
            }
            
            BoltDeclarer d = builder.setBolt(id, new TridentBoltExecutor(c.bolt, batchIdsForBolts, specs), c.parallelism);
            for(Map conf: c.componentConfs) {
                d.addConfigurations(conf);

View Full Code Here

0 1 2 3 4

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.