Examples of StormClusterState


Examples of com.alibaba.jstorm.cluster.StormClusterState

   *     skip check
   * @param data
   */
  @Override
  public void run() {
    StormClusterState clusterState = data.getStormClusterState();

    try {
      // Attetion, need first check Assignments
      List<String> active_topologys = clusterState.assignments(null);

      if (active_topologys == null) {
        LOG.info("Failed to get active topologies");
        return;
      }

      for (String topologyid : active_topologys) {

        LOG.debug("Check tasks " + topologyid);

        // Attention, here don't check /ZK-dir/taskbeats/topologyid to
        // get task ids
        List<Integer> taskIds = clusterState.task_ids(topologyid);
        if (taskIds == null) {
          LOG.info("Failed to get task ids of " + topologyid);
          continue;
        }
        Assignment assignment = clusterState.assignment_info(
            topologyid, null);

        boolean needReassign = false;
        for (Integer task : taskIds) {
          boolean isTaskDead = NimbusUtils.isTaskDead(data,
              topologyid, task);
          if (isTaskDead == true) {
            LOG.info("Found " + topologyid + ",taskid:" + task
                + " is dead");
           
            ResourceWorkerSlot resource = null;
            if (assignment != null)
              resource = assignment.getWorkerByTaskId(task);
            if (resource != null) {
              Date now = new Date();
              String nowStr = TimeFormat.getSecond(now);
              String errorInfo = "Task-" + task + " is dead on "
                  + resource.getHostname() + ":"
                  + resource.getPort() + ", " + nowStr;
              LOG.info(errorInfo);
                clusterState.report_task_error(topologyid, task, errorInfo);
            }
            needReassign = true;
          }
        }
        if (needReassign == true) {
View Full Code Here

Examples of com.alibaba.jstorm.cluster.StormClusterState

    /*
     * Step 2: create ZK operation instance StromClusterState
     */

    StormClusterState stormClusterState = Cluster
        .mk_storm_cluster_state(conf);

    /*
     * Step 3, create LocalStat LocalStat is one KV database 4.1 create
     * LocalState instance 4.2 get supervisorId, if no supervisorId, create
View Full Code Here

Examples of com.alibaba.jstorm.cluster.StormClusterState

      updateWorkerMetricsToZK(workerMetricList, metricPerf);
      updateUserDefMetricsToZK(metricPerf);
    }
   
    private void updateTaskMetricsToZK(Map<String, List<MetricInfo>> metricMap, boolean metricPerf) {
        StormClusterState clusterState = workerData.getZkCluster();
      String topologyId = workerData.getTopologyId();
     
      for(Entry<String, List<MetricInfo>> entry : metricMap.entrySet()) {
        String taskId = entry.getKey();
        List<MetricInfo> MetricList = entry.getValue();
       
        try {
          String component = clusterState.task_info(topologyId, Integer.valueOf(taskId)).getComponentId();
            TaskMetricInfo taskMetricInfo = new TaskMetricInfo(taskId, component);
           
              for(MetricInfo metricInfo : MetricList) {
              taskMetricInfo.updateMetricData(metricInfo);
              }
             
              List<String> errors = taskMetricInfo.anyQueueFull();
              if (errors.size() > 0) {
                for (String error : errors)
                      clusterState.report_task_error(topologyId, Integer.valueOf(taskId), error);
              }
             
            clusterState.update_task_metric(topologyId, taskId, taskMetricInfo);
        } catch(Exception e) {
          logger.error(marker, "Failed to update metrics data in ZK for topo-{} task-{}.",
                    topologyId, taskId, e);
      }
      }
View Full Code Here

Examples of com.alibaba.jstorm.cluster.StormClusterState

   
    return value;
  }
   
    private void updateWorkerMetricsToZK(List<MetricInfo> metricList, boolean metricPerf) {
        StormClusterState clusterState = workerData.getZkCluster();
      String topologyId = workerData.getTopologyId();
      String hostName;

      hostName = JStormServerUtils.getHostName(workerData.getConf());
      String workerId = hostName + ":" + workerData.getPort();
     
      WorkerMetricInfo workerMetricInfo = new WorkerMetricInfo(hostName, workerData.getPort());
      try {
        //Set metrics data
          for(MetricInfo metricInfo : metricList) {   
          workerMetricInfo.updateMetricData(metricInfo)
        }
         
          //Set cpu & memory usage
          Runtime rt=Runtime.getRuntime();
          long usedMem = rt.totalMemory() - rt.freeMemory();
          workerMetricInfo.setUsedMem(usedMem);  
         
          workerMetricInfo.setUsedCpu(getCpuUsage());
             
          clusterState.update_worker_metric(topologyId, workerId, workerMetricInfo);
      } catch(Exception e) {
        logger.error(marker, "Failed to update metrics data in ZK for topo-{} idStr-{}.",
                topologyId, workerId, e);
      }
    }
View Full Code Here

Examples of com.alibaba.jstorm.cluster.StormClusterState

                topologyId, workerId, e);
      }
    }
   
    private void updateUserDefMetricsToZK(boolean metricPerf) {
      StormClusterState clusterState = workerData.getZkCluster();
      String topologyId = workerData.getTopologyId();
      String hostName =JStormServerUtils.getHostName(workerData.getConf());
      String workerId = hostName + ":" + workerData.getPort();
     
      UserDefMetric userDefMetric = Metrics.getUserDefMetric();
      UserDefMetricData userDefMetricData = new UserDefMetricData();
      userDefMetricData.updateFromGauge(userDefMetric.getGauge());
      userDefMetricData.updateFromCounter(userDefMetric.getCounter());
      userDefMetricData.updateFromMeterData(userDefMetric.getMeter());
      // If metrics performance is disable, Timer & Histogram metrics will not be monitored,
      // and the corresponding metrics data will not be sent to ZK either.
      if (metricPerf == false) {
         userDefMetricData.updateFromHistogramData(userDefMetric.getHistogram());
         userDefMetricData.updateFromTimerData(userDefMetric.getTimer());
      }

      try {
          clusterState.update_userDef_metric(topologyId, workerId, userDefMetricData);
      } catch(Exception e) {
        logger.error(marker, "Failed to update user define metrics data in ZK for topo-{} idStr-{}.",
                topologyId, workerId, e);
      }
     
View Full Code Here

Examples of com.alipay.bluewhale.core.cluster.StormClusterState



  public static <T> void transitionName(NimbusData data, String topologyName,
    boolean errorOnNoTransition, StatusType transition_status, T... args) throws NotAliveException {
      StormClusterState stormClusterState = data.getStormClusterState();
      String topologyId = Common.get_storm_id(stormClusterState, topologyName);
      if (topologyId == null){
    throw new NotAliveException(topologyName);
      }
      transition(data, topologyId, errorOnNoTransition, transition_status, args);
View Full Code Here

Examples of com.alipay.bluewhale.core.cluster.StormClusterState

    StormTopology newtopology = new StormTopology(topology);
    // TODO TOPOLOGY_OPTIMIZE
    // if ((Boolean) totalStormConf.get(Config.TOPOLOGY_OPTIMIZE)) {
    // newtopology = optimizeTopology(topology);
    // }
    StormClusterState stormClusterState = data.getStormClusterState();
    // this validates the structure of the topology
    Common.system_topology(totalStormConf, newtopology);
    LOG.info("Received topology submission for " + topologyname
        + " with conf " + serializedConf);
    synchronized (data.getSubmitLock()) {
      try {
        // �ϴ�code�������л�topology��������Ϣ
        setupStormCode(conf, topologyId, uploadedJarLocation,
            serializedConf, newtopology);
        // ��������
        stormClusterState.setup_heartbeats(topologyId);
        //Ϊtask�����Ӧcomponentid
        setupStormStatic(conf, topologyId, stormClusterState);
        //make assignments for a topology
        NimbusUtils.mkAssignments(data, topologyId);
        //����topology������״̬Ϊ:active
View Full Code Here

Examples of com.alipay.bluewhale.core.cluster.StormClusterState

   *  ����topology����Ϣ��topologyid��topologyname����������worker��������ʱ���״̬
   *  @return ClusterSummary
         */
  @Override
  public ClusterSummary getClusterInfo() throws TException {
    StormClusterState stormClusterState = data.getStormClusterState();
    //����ѷ���Ķ˿ںͶ˿������е�����
    Map<String, Set<Integer>> assigned = NimbusUtils
        .assigned_Slots(stormClusterState);
    //�������supervisor����Ϣ
    Map<String, SupervisorInfo> supervisorInfos = NimbusUtils
        .allSupervisorInfo(stormClusterState, null);
    //���supervisorSummaries
    List<SupervisorSummary> supervisorSummaries = new ArrayList<SupervisorSummary>();
    if (supervisorInfos == null) {
      supervisorInfos = new HashMap<String, SupervisorInfo>();
    }
    Set<Entry<String, SupervisorInfo>> sinfoEntry = supervisorInfos.entrySet();
    for (Iterator<Entry<String, SupervisorInfo>> it = sinfoEntry.iterator(); it.hasNext();) {
      Entry<String, SupervisorInfo> entry = it.next();
      String supervisorId = entry.getKey();
      SupervisorInfo info = entry.getValue();
     
      List<Integer> ports = info.getWorkPorts();
      int num_used_workers = 0;
      int num_workers = 0;
      if (assigned != null && assigned.get(supervisorId) != null){
          num_used_workers = assigned.get(supervisorId).size();
      }
                        if (ports != null){
                            num_workers = ports.size();
      }
      supervisorSummaries.add(new SupervisorSummary(info.getHostName(),
          info.getUptimeSecs(),num_workers , num_used_workers));
    }
    //���nimbus������ʱ��
    int uptime = data.uptime();
   
    //�������״̬Ϊactive��topology
    Map<String, StormBase> bases = Common.topology_bases(stormClusterState);

    if (bases == null) {
      bases = new HashMap<String, StormBase>();
    }

    //������м�Ⱥ����Ϣ
    List<TopologySummary> topologySummaries = new ArrayList<TopologySummary>();

    Set<Entry<String, StormBase>> basesEntry = bases.entrySet();
    for (Iterator<Entry<String, StormBase>> it = basesEntry.iterator(); it.hasNext();) {
      Entry<String, StormBase> entry = it.next();
      String stormId = entry.getKey();
      StormBase base = entry.getValue();
      Assignment assignment = stormClusterState.assignment_info(stormId,null);
      if (assignment != null) {
        HashSet<NodePort> workers = new HashSet<NodePort>();
        Collection<NodePort> entryColl = assignment.getTaskToNodeport().values();
        workers.addAll(entryColl);
        topologySummaries.add(new TopologySummary(stormId, base.getStormName(), assignment.getTaskToNodeport().size(),
View Full Code Here

Examples of com.alipay.bluewhale.core.cluster.StormClusterState

  @Override
  public TopologyInfo getTopologyInfo(String topologyIdthrows NotAliveException, TException {
   
    TopologyInfo topologyInfo=null;
   
    StormClusterState stormClusterState = data.getStormClusterState();
   
    //���topology��������Ϣ�� ����ID �� componentid
    HashMap<Integer, String> taskInfo = Common.topology_task_info(stormClusterState, topologyId);// <taskid,componentid>
   
    //���topology��name������ʱ���״̬
    StormBase base = stormClusterState.storm_base(topologyId, null);
   
    //���topology�����������Ϣ
    Assignment assignment = stormClusterState.assignment_info(topologyId,null);
   
    if (base != null && assignment != null) {
      List<TaskSummary> taskSummarys = new ArrayList<TaskSummary>();
      Set<Entry<Integer, String>> taskInfoSet = taskInfo.entrySet();
      for (Iterator<Entry<Integer, String>> it = taskInfoSet.iterator(); it.hasNext();) {
        Entry<Integer, String> entry =  it.next();
        Integer taskId =  entry.getKey();
        String componentId =  entry.getValue();
        NodePort np = (NodePort) assignment.getTaskToNodeport().get(taskId);
        //���ָ��task��������Ϣ
        TaskHeartbeat heartbeat = stormClusterState.task_heartbeat(topologyId, taskId);
        if (np == null || heartbeat == null) {
          continue;
        }
        String host = (String) assignment.getNodeHost().get(np.getNode());
        List<TaskError> errors = stormClusterState.task_errors(topologyId, taskId);
        List<ErrorInfo> newErrors = new ArrayList<ErrorInfo>();

        if (errors != null) {
          int size = errors.size();
          for (int i = 0; i < size; i++) {
View Full Code Here

Examples of com.alipay.bluewhale.core.cluster.StormClusterState

  }

  TimeCacheMap<String, Long> lastneedClean=new TimeCacheMap<String, Long>(3600);
  @Override
  public void run() {
    StormClusterState clusterState = data.getStormClusterState();

    List<String> active_topologys = clusterState.active_storms();
    if (active_topologys != null){
        for (String topologyid : active_topologys) {
      StatusTransition.transition(data, topologyid, false, StatusType.monitor);
        }
    }
   
    long times=index.incrementAndGet();
    if(times%10!=0)
    {
      return ;
    }
    if(times>10000000)
    {
      index.set(0l);
    }


    // �̰߳�ȫ��������submitTopology��transition״̬ʱ
    synchronized (data.getSubmitLock()) {
      Set<String> to_cleanup_ids = do_cleanup(clusterState,active_topologys);
      if (to_cleanup_ids != null && to_cleanup_ids.size() > 0) {
        for (String storm_id : to_cleanup_ids) {
          if(!lastneedClean.containsKey(storm_id))
          {
              LOG.info("nexttime Cleaning up " + storm_id);
              lastneedClean.put(storm_id, System.currentTimeMillis() );
            continue;
          }
            LOG.info("Cleaning up " + storm_id);
          clusterState.teardown_heartbeats(storm_id);
          clusterState.teardown_task_errors(storm_id);
          // ��ȡĿ¼ /nimbus/stormdist/topologyid
          String master_stormdist_root = StormConfig
              .masterStormdistRoot(data.getConf(), storm_id);
          try {
            // ǿ��ɾ��topologyid��Ӧ��Ŀ¼
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.