Package com.linkedin.helix.model

Examples of com.linkedin.helix.model.IdealState


  private static List<Message> computeMessagesForSimpleTransition(ZNRecord idealStateRecord)
  {
    List<Message> msgList = new ArrayList<Message>();

    IdealState idealState = new IdealState(idealStateRecord);
    for (String stateUnitKey : idealState.getPartitionSet())
    {
      Map<String, String> instanceStateMap;
      instanceStateMap = idealState.getInstanceStateMap(stateUnitKey);
    }

    return msgList;
  }
View Full Code Here


                                         String resourceName,
                                         int replica)
  {
    List<String> nodeNames = _mgmtTool.getInstancesInCluster(clusterName);

    IdealState idealState = _mgmtTool.getResourceIdealState(clusterName, resourceName);
    idealState.setReplicas(Integer.toString(replica));
    int partitions = idealState.getNumPartitions();

    ZNRecord newIdealState =
        IdealStateCalculatorForStorageNode.calculateIdealState(nodeNames,
                                                               partitions,
                                                               replica - 1,
                                                               resourceName,
                                                               "MASTER",
                                                               "SLAVE");

    newIdealState.merge(idealState.getRecord());
    _mgmtTool.setResourceIdealState(clusterName,
                                    resourceName,
                                    new IdealState(newIdealState));
  }
View Full Code Here

    if(instances.size() % replica != 0)
    {
      throw new HelixException("Instances must be divided by replica");
    }
   
    IdealState result = new IdealState(resultRecordName);
    result.setNumPartitions(partitions.size());
    result.setReplicas("" + replica);
    result.setStateModelDefRef(stateModelName);
   
    int groups = instances.size() / replica;
    int remainder = instances.size() % replica;
   
    int remainder2 = partitions.size() % groups;
    int storageNodeGroupSize = partitions.size() / groups;
   
    for(int i = 0; i < groups; i++)
    {
      int relayStart = 0, relayEnd = 0, storageNodeStart = 0, storageNodeEnd = 0;
      if(i < remainder)
      {
        relayStart = (replica + 1) * i;
        relayEnd = (replica + 1) *(i + 1);
      }
      else
      {
        relayStart = (replica + 1) * remainder + replica * (i - remainder);
        relayEnd = relayStart + replica;
      }
      //System.out.println("relay start :" + relayStart + " relayEnd:" + relayEnd);
      if(i < remainder2)
      {
        storageNodeStart = (storageNodeGroupSize + 1) * i;
        storageNodeEnd = (storageNodeGroupSize + 1) *(i + 1);
      }
      else
      {
        storageNodeStart = (storageNodeGroupSize + 1) * remainder2 + storageNodeGroupSize * (i - remainder2);
        storageNodeEnd = storageNodeStart + storageNodeGroupSize;
      }

      //System.out.println("storageNodeStart :" + storageNodeStart + " storageNodeEnd:" + storageNodeEnd);
      List<String> snBatch = partitions.subList(storageNodeStart, storageNodeEnd);
      List<String> relayBatch = instances.subList(relayStart, relayEnd);
     
      Map<String, List<String>> sublistFields = calculateSubIdealState(snBatch, relayBatch, replica);
     
      result.getRecord().getListFields().putAll(sublistFields);
    }
   
    for(String snName : result.getRecord().getListFields().keySet())
    {
      Map<String, String> mapField = new TreeMap<String, String>();
      List<String> relayCandidates = result.getRecord().getListField(snName);
      mapField.put(relayCandidates.get(0), firstValue);
      for(int i = 1; i < relayCandidates.size(); i++)
      {
        mapField.put(relayCandidates.get(i), restValue);
      }
      result.getRecord().getMapFields().put(snName, mapField);
    }
    System.out.println();
    return result;
  }
View Full Code Here

  public void testNewMasterSelection() throws Exception
  {
    HelixManager manager = ((TestHelper.StartCMResult) (_startCMResultMap.values().toArray()[0]))._manager;
    HelixDataAccessor accessor = manager.getHelixDataAccessor();

    IdealState resourceIdealState = manager.getClusterManagmentTool().getResourceIdealState(manager.getClusterName(), TEST_DB);
    Map<String, Map<String, Integer>> scnMap = new HashMap<String, Map<String, Integer>>();

    List<String> instancesInCluster = manager.getClusterManagmentTool().getInstancesInCluster(manager.getClusterName());
    List<String> instances = new ArrayList<String>();
    instances.addAll(instancesInCluster);
    //instances.add(instancesInCluster.get(0));
    instances.add("deadInstance");
    System.out.println(instances.size());

    int seq = 50;
    for(String instance : instances)
    {
      ZNRecord scnRecord = new ZNRecord("scnTable");
      scnRecord.setSimpleField("instance", instance);
      for(int i = 0; i < _PARTITIONS; i++)
      {
        Map<String, String> scnDetails = new HashMap<String, String>();

        String partition = TEST_DB + "_" + i;
        List<String> idealStatePrefList =
            resourceIdealState.getPreferenceList(partition);
        String idealStateMaster = idealStatePrefList.get(0);

        scnDetails.put("gen", "4");

        if (instance.equals(idealStateMaster))
View Full Code Here

      throw new HelixException("Cluster: " + clusterName + ", resource: " + resourceName
          + ", ideal state does not exist");
    }

    // check partitions exist. warn if not
    IdealState idealState = new IdealState(idealStateRecord);
    for (String partitionName : partitionNames)
    {
      if ((idealState.getIdealStateMode() == IdealStateModeProperty.AUTO && idealState.getPreferenceList(partitionName) == null)
          || (idealState.getIdealStateMode() == IdealStateModeProperty.CUSTOMIZED && idealState.getInstanceStateMap(partitionName) == null))
      {
        logger.warn("Cluster: " + clusterName + ", resource: " + resourceName
            + ", partition: " + partitionName
            + ", partition does not exist in ideal state");
      }
View Full Code Here

          + partitionNames + " on " + instanceName + ", because " + instanceName
          + " is not alive");
    }

    // check resource group exists
    IdealState idealState = accessor.getProperty(keyBuilder.idealStates(resourceName));
    if (idealState == null)
    {
      throw new HelixException("Can't reset state for " + resourceName + "/"
          + partitionNames + " on " + instanceName + ", because " + resourceName
          + " is not added");
    }

    // check partition exists in resource group
    Set<String> resetPartitionNames = new HashSet<String>(partitionNames);
    if (idealState.getIdealStateMode() == IdealStateModeProperty.CUSTOMIZED)
    {
      Set<String> partitions =
          new HashSet<String>(idealState.getRecord().getMapFields().keySet());
      if (!partitions.containsAll(resetPartitionNames))
      {
        throw new HelixException("Can't reset state for " + resourceName + "/"
            + partitionNames + " on " + instanceName + ", because not all "
            + partitionNames + " exist");
      }
    }
    else
    {
      Set<String> partitions =
          new HashSet<String>(idealState.getRecord().getListFields().keySet());
      if (!partitions.containsAll(resetPartitionNames))
      {
        throw new HelixException("Can't reset state for " + resourceName + "/"
            + partitionNames + " on " + instanceName + ", because not all "
            + partitionNames + " exist");
      }
    }

    // check partition is in ERROR state
    String sessionId = liveInstance.getSessionId();
    CurrentState curState =
        accessor.getProperty(keyBuilder.currentState(instanceName,
                                                     sessionId,
                                                     resourceName));
    for (String partitionName : resetPartitionNames)
    {
      if (!curState.getState(partitionName).equals("ERROR"))
      {
        throw new HelixException("Can't reset state for " + resourceName + "/"
            + partitionNames + " on " + instanceName + ", because not all "
            + partitionNames + " are in ERROR state");
      }
    }

    // check stateModelDef exists and get initial state
    String stateModelDef = idealState.getStateModelDefRef();
    StateModelDefinition stateModel =
        accessor.getProperty(keyBuilder.stateModelDef(stateModelDef));
    if (stateModel == null)
    {
      throw new HelixException("Can't reset state for " + resourceName + "/"
          + partitionNames + " on " + instanceName + ", because " + stateModelDef
          + " is NOT found");
    }

    // check there is no pending messages for the partitions exist
    List<Message> messages = accessor.getChildValues(keyBuilder.messages(instanceName));
    for (Message message : messages)
    {
      if (!MessageType.STATE_TRANSITION.toString().equalsIgnoreCase(message.getMsgType())
          || !sessionId.equals(message.getTgtSessionId())
          || !resourceName.equals(message.getResourceName())
          || !resetPartitionNames.contains(message.getPartitionName()))
      {
        continue;
      }

      throw new HelixException("Can't reset state for " + resourceName + "/"
          + partitionNames + " on " + instanceName
          + ", because a pending message exists: " + message);
    }

    String adminName = null;
    try
    {
      adminName = InetAddress.getLocalHost().getCanonicalHostName() + "-ADMIN";
    }
    catch (UnknownHostException e)
    {
      // can ignore it
      logger.info("Unable to get host name. Will set it to UNKNOWN, mostly ignorable", e);
      adminName = "UNKNOWN";
    }

    List<Message> resetMessages = new ArrayList<Message>();
    List<PropertyKey> messageKeys = new ArrayList<PropertyKey>();
    for (String partitionName : resetPartitionNames)
    {
      // send ERROR to initialState message
      String msgId = UUID.randomUUID().toString();
      Message message = new Message(MessageType.STATE_TRANSITION, msgId);
      message.setSrcName(adminName);
      message.setTgtName(instanceName);
      message.setMsgState(MessageState.NEW);
      message.setPartitionName(partitionName);
      message.setResourceName(resourceName);
      message.setTgtSessionId(sessionId);
      message.setStateModelDef(stateModelDef);
      message.setFromState("ERROR");
      message.setToState(stateModel.getInitialState());
      message.setStateModelFactoryName(idealState.getStateModelFactoryName());

      resetMessages.add(message);
      messageKeys.add(keyBuilder.message(instanceName, message.getId()));
    }
View Full Code Here

    }
    catch (Exception e)
    {
      logger.error("", e);
    }
    IdealState idealState = new IdealState(resourceName);
    idealState.setNumPartitions(partitions);
    idealState.setStateModelDefRef(stateModelRef);
    idealState.setIdealStateMode(mode.toString());
    idealState.setReplicas("" + 0);
    idealState.setStateModelFactoryName(HelixConstants.DEFAULT_STATE_MODEL_FACTORY);

    if (bucketSize > 0)
    {
      idealState.setBucketSize(bucketSize);
    }

    String stateModelDefPath =
        PropertyPathConfig.getPath(PropertyType.STATEMODELDEFS,
                                   clusterName,
                                   stateModelRef);
    if (!_zkClient.exists(stateModelDefPath))
    {
      throw new HelixException("State model " + stateModelRef
          + " not found in the cluster STATEMODELDEFS path");
    }

    String idealStatePath = HelixUtil.getIdealStatePath(clusterName);
    String dbIdealStatePath = idealStatePath + "/" + resourceName;
    if (_zkClient.exists(dbIdealStatePath))
    {
      throw new HelixException("Skip the operation. DB ideal state directory exists:"
          + dbIdealStatePath);
    }

    ZKUtil.createChildren(_zkClient, idealStatePath, idealState.getRecord());
  }
View Full Code Here

    if (!ZKUtil.isClusterSetup(clusterName, _zkClient))
    {
      throw new HelixException("Cluster " + clusterName + " is not setup yet");
    }

    IdealState idealState = new IdealState(clusterName);

    idealState.setNumPartitions(1);
    idealState.setStateModelDefRef("LeaderStandby");

    List<String> controllers = getInstancesInCluster(grandCluster);
    if (controllers.size() == 0)
    {
      throw new HelixException("Grand cluster " + grandCluster + " has no instances");
    }
    idealState.setReplicas(Integer.toString(controllers.size()));
    Collections.shuffle(controllers);
    idealState.getRecord().setListField(clusterName, controllers);
    idealState.setPartitionState(clusterName, controllers.get(0), "LEADER");
    for (int i = 1; i < controllers.size(); i++)
    {
      idealState.setPartitionState(clusterName, controllers.get(i), "STANDBY");
    }

    ZKHelixDataAccessor accessor =
        new ZKHelixDataAccessor(grandCluster, new ZkBaseDataAccessor<ZNRecord>(_zkClient));
    Builder keyBuilder = accessor.keyBuilder();

    accessor.setProperty(keyBuilder.idealStates(idealState.getResourceName()), idealState);
  }
View Full Code Here

    List<String> InstanceNames = getInstancesInCluster(clusterName);

    // ensure we get the same idealState with the same set of instances
    Collections.sort(InstanceNames);

    IdealState idealState = getResourceIdealState(clusterName, resourceName);
    if (idealState == null)
    {
      throw new HelixException("Resource: " + resourceName + " has NOT been added yet");
    }

    idealState.setReplicas(Integer.toString(replica));
    int partitions = idealState.getNumPartitions();
    String stateModelName = idealState.getStateModelDefRef();
    StateModelDefinition stateModDef = getStateModelDef(clusterName, stateModelName);

    if (stateModDef == null)
    {
      throw new HelixException("cannot find state model: " + stateModelName);
    }
    // StateModelDefinition def = new StateModelDefinition(stateModDef);

    List<String> statePriorityList = stateModDef.getStatesPriorityList();

    String masterStateValue = null;
    String slaveStateValue = null;
    replica--;

    for (String state : statePriorityList)
    {
      String count = stateModDef.getNumInstancesPerState(state);
      if (count.equals("1"))
      {
        if (masterStateValue != null)
        {
          throw new HelixException("Invalid or unsupported state model definition");
        }
        masterStateValue = state;
      }
      else if (count.equalsIgnoreCase("R"))
      {
        if (slaveStateValue != null)
        {
          throw new HelixException("Invalid or unsupported state model definition");
        }
        slaveStateValue = state;
      }
      else if (count.equalsIgnoreCase("N"))
      {
        if (!(masterStateValue == null && slaveStateValue == null))
        {
          throw new HelixException("Invalid or unsupported state model definition");
        }
        replica = InstanceNames.size() - 1;
        masterStateValue = slaveStateValue = state;
      }
    }
    if (masterStateValue == null && slaveStateValue == null)
    {
      throw new HelixException("Invalid or unsupported state model definition");
    }

    if (masterStateValue == null)
    {
      masterStateValue = slaveStateValue;
    }
    if (idealState.getIdealStateMode() != IdealStateModeProperty.AUTO_REBALANCE)
    {
      ZNRecord newIdealState =
          IdealStateCalculatorForStorageNode.calculateIdealState(InstanceNames,
                                                                 partitions,
                                                                 replica,
                                                                 keyPrefix,
                                                                 masterStateValue,
                                                                 slaveStateValue);

      // for now keep mapField in AUTO mode and remove listField in CUSTOMIZED mode
      if (idealState.getIdealStateMode() == IdealStateModeProperty.AUTO)
      {
        idealState.getRecord().setListFields(newIdealState.getListFields());
        idealState.getRecord().setMapFields(newIdealState.getMapFields());
      }
      if (idealState.getIdealStateMode() == IdealStateModeProperty.CUSTOMIZED)
      {
        idealState.getRecord().setMapFields(newIdealState.getMapFields());
      }
    }
    else
    {
      for (int i = 0; i < partitions; i++)
      {
        String partitionName = keyPrefix + "_" + i;
        idealState.getRecord().setMapField(partitionName, new HashMap<String, String>());
        idealState.getRecord().setListField(partitionName, new ArrayList<String>());
      }
    }
    setResourceIdealState(clusterName, resourceName, idealState);
  }
View Full Code Here

    if (idealStateRecord.getId() == null
        || !idealStateRecord.getId().equals(resourceName))
    {
      throw new IllegalArgumentException("ideal state must have same id as resource name");
    }
    setResourceIdealState(clusterName, resourceName, new IdealState(idealStateRecord));
  }
View Full Code Here

TOP

Related Classes of com.linkedin.helix.model.IdealState

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.