Examples of org.apache.helix.HelixManager

org.apache.helix.HelixManager

First class Object any process will interact with
General flow

 manager = HelixManagerFactory.getManagerFor(); ROLE can be participant, spectator or a controller
 manager.connect(); manager.addSOMEListener(); manager.start() After start is invoked the subsequent interactions will be via listener onChange callbacks There will be 3 scenarios for onChange callback, which can be determined using NotificationContext.type INIT -> will be invoked the first time the listener is added CALLBACK -> will be invoked due to datachange in the property value FINALIZE -> will be invoked when listener is removed or session expires manager.disconnect()

Default implementations available @see HelixStateMachineEngine for participant @see RoutingTableProvider for spectator @see GenericHelixController for controller

    final String clusterName = "lock-manager-demo";
    final String lockGroupName = "lock-group";
    final int numInstances = 3;
    final int numPartitions = 12;
    final boolean startController = false;
    HelixManager controllerManager = null;
    Thread[] processArray;
    processArray = new Thread[numInstances];
    try
    {
      startLocalZookeeper(2199);
      HelixAdmin admin = new ZKHelixAdmin(zkAddress);
      admin.addCluster(clusterName, true);
      StateModelConfigGenerator generator = new StateModelConfigGenerator();
      admin.addStateModelDef(clusterName, "OnlineOffline",
          new StateModelDefinition(generator.generateConfigForOnlineOffline()));
      admin.addResource(clusterName, lockGroupName, numPartitions,
          "OnlineOffline", IdealStateModeProperty.AUTO_REBALANCE.toString());
      admin.rebalance(clusterName, lockGroupName, 1);
      for (int i = 0; i < numInstances; i++)
      {
        final String instanceName = "localhost_" + (12000 + i);
        processArray[i] = new Thread(new Runnable()
        {


          @Override
          public void run()
          {
            LockProcess lockProcess = null;


            try
            {
              lockProcess = new LockProcess(clusterName, zkAddress,
                  instanceName, startController);
              lockProcess.start();
              Thread.currentThread().join();
            } catch (InterruptedException e)
            {
              System.out.println(instanceName + "Interrupted");
              if (lockProcess != null)
              {
                lockProcess.stop();
              }
            } catch (Exception e)
            {
              e.printStackTrace();
            }
          }


        });
        processArray[i].start();
      }
      Thread.sleep(3000);
      controllerManager = HelixControllerMain.startHelixController(zkAddress,
          clusterName, "controller", HelixControllerMain.STANDALONE);
      Thread.sleep(5000);
      printStatus(admin, clusterName, lockGroupName);
      System.out.println("Stopping localhost_12000");
      processArray[0].interrupt();
      Thread.sleep(3000);
      printStatus(admin, clusterName, lockGroupName);
      Thread.currentThread().join();
    } catch (Exception e)
    {
      e.printStackTrace();
    } finally
    {
      if (controllerManager != null)
      {
        controllerManager.disconnect();
      }
      for (Thread process : processArray)
      {
        if (process != null)
        {

View Full Code Here

  @Override
  public void process(ClusterEvent event) throws Exception {
    long startTime = System.currentTimeMillis();
    logger.info("START ReadClusterDataStage.process()");


    HelixManager manager = event.getAttribute("helixmanager");
    if (manager == null) {
      throw new StageException("HelixManager attribute value is null");
    }
    HelixDataAccessor dataAccessor = manager.getHelixDataAccessor();
    _cache.refresh(dataAccessor);


    ClusterStatusMonitor clusterStatusMonitor =
        (ClusterStatusMonitor) event.getAttribute("clusterStatusMonitor");
    if (clusterStatusMonitor != null) {

View Full Code Here

    }
  }


  @Test()
  public void TestMessageSend() {
    HelixManager manager = new MockHelixManager();
    DefaultMessagingService svc = new DefaultMessagingService(manager);
    TestMessageHandlerFactory factory = new TestMessageHandlerFactory();
    svc.registerMessageHandlerFactory(factory.getMessageType(), factory);


    Criteria recipientCriteria = new Criteria();

View Full Code Here

  @Override
  public void process(ClusterEvent event) throws Exception {
    long startTime = System.currentTimeMillis();
    logger.info("START TaskAssignmentStage.process()");


    HelixManager manager = event.getAttribute("helixmanager");
    Map<String, Resource> resourceMap = event.getAttribute(AttributeName.RESOURCES.toString());
    MessageThrottleStageOutput messageOutput =
        event.getAttribute(AttributeName.MESSAGES_THROTTLE.toString());
    ClusterDataCache cache = event.getAttribute("ClusterDataCache");
    Map<String, LiveInstance> liveInstanceMap = cache.getLiveInstances();


    if (manager == null || resourceMap == null || messageOutput == null || cache == null
        || liveInstanceMap == null) {
      throw new StageException("Missing attributes in event:" + event
          + ". Requires HelixManager|RESOURCES|MESSAGES_THROTTLE|DataCache|liveInstanceMap");
    }


    HelixDataAccessor dataAccessor = manager.getHelixDataAccessor();
    List<Message> messagesToSend = new ArrayList<Message>();
    for (String resourceName : resourceMap.keySet()) {
      Resource resource = resourceMap.get(resourceName);
      for (Partition partition : resource.getPartitions()) {
        List<Message> messages = messageOutput.getMessages(resourceName, partition);
        messagesToSend.addAll(messages);
      }
    }


    List<Message> outputMessages =
        batchMessage(dataAccessor.keyBuilder(), messagesToSend, resourceMap, liveInstanceMap,
            manager.getProperties());
    sendMessages(dataAccessor, outputMessages);


    long endTime = System.currentTimeMillis();
    logger.info("END TaskAssignmentStage.process(). took: " + (endTime - startTime) + " ms");

View Full Code Here

   * lock-always: caller always needs to obtain an external lock before call, calls to
   * handleEvent() should be serialized
   * @param event
   */
  protected synchronized void handleEvent(ClusterEvent event) {
    HelixManager manager = event.getAttribute("helixmanager");
    if (manager == null) {
      logger.error("No cluster manager in event:" + event.getName());
      return;
    }


    if (!manager.isLeader()) {
      logger.error("Cluster manager: " + manager.getInstanceName()
          + " is not leader. Pipeline will not be invoked");
      return;
    }


    if (_paused) {
      logger.info("Cluster is paused. Ignoring the event:" + event.getName());
      return;
    }


    NotificationContext context = null;
    if (event.getAttribute("changeContext") != null) {
      context = (NotificationContext) (event.getAttribute("changeContext"));
    }


    // Initialize _clusterStatusMonitor
    if (context != null) {
      if (context.getType() == Type.FINALIZE) {
        if (_clusterStatusMonitor != null) {
          _clusterStatusMonitor.reset();
          _clusterStatusMonitor = null;
        }


        stopRebalancingTimer();
        logger.info("Get FINALIZE notification, skip the pipeline. Event :" + event.getName());
        return;
      } else {
        if (_clusterStatusMonitor == null) {
          _clusterStatusMonitor = new ClusterStatusMonitor(manager.getClusterName());
        }


        event.addAttribute("clusterStatusMonitor", _clusterStatusMonitor);
      }
    }

View Full Code Here

    }


    Map<String, LiveInstance> lastInstances = _lastSeenInstances.get();
    Map<String, LiveInstance> lastSessions = _lastSeenSessions.get();


    HelixManager manager = changeContext.getManager();
    Builder keyBuilder = new Builder(manager.getClusterName());
    if (lastSessions != null) {
      for (String session : lastSessions.keySet()) {
        if (!curSessions.containsKey(session)) {
          // remove current-state listener for expired session
          String instanceName = lastSessions.get(session).getInstanceName();
          manager.removeListener(keyBuilder.currentStates(instanceName, session), this);
        }
      }
    }


    if (lastInstances != null) {
      for (String instance : lastInstances.keySet()) {
        if (!curInstances.containsKey(instance)) {
          // remove message listener for disconnected instances
          manager.removeListener(keyBuilder.messages(instance), this);
        }
      }
    }


    for (String session : curSessions.keySet()) {
      if (lastSessions == null || !lastSessions.containsKey(session)) {
        String instanceName = curSessions.get(session).getInstanceName();
        try {
          // add current-state listeners for new sessions
          manager.addCurrentStateChangeListener(this, instanceName, session);
          logger.info(manager.getInstanceName() + " added current-state listener for instance: "
              + instanceName + ", session: " + session + ", listener: " + this);
        } catch (Exception e) {
          logger.error("Fail to add current state listener for instance: " + instanceName
              + " with session: " + session, e);
        }
      }
    }


    for (String instance : curInstances.keySet()) {
      if (lastInstances == null || !lastInstances.containsKey(instance)) {
        try {
          // add message listeners for new instances
          manager.addMessageListener(this, instance);
          logger.info(manager.getInstanceName() + " added message listener for " + instance
              + ", listener: " + this);
        } catch (Exception e) {
          logger.error("Fail to add message listener for instance: " + instance, e);
        }
      }

View Full Code Here

public class TestJosqlProcessor extends ZkStandAloneCMTestBase {
  @Test(groups = {
    "integrationTest"
  })
  public void testJosqlQuery() throws Exception {
    HelixManager manager = _participants[0];
    // ((TestHelper.StartCMResult) (_startCMResultMap.values().toArray()[0]))._manager;


    // Find the instance name that contains partition TestDB_2 and state is 'MASTER'
    String SQL =
        "SELECT id  "

View Full Code Here

    Assert.assertTrue(exceptionThrown);
  }


  @Test(groups = ("unitTest"))
  public void testOrderby() throws Exception {
    HelixManager manager = _participants[0];
    // ((TestHelper.StartCMResult) (_startCMResultMap.values().toArray()[0]))._manager;


    Map<String, ZNRecord> scnMap = new HashMap<String, ZNRecord>();
    for (int i = 0; i < NODE_NR; i++) {
      String instance = "localhost_" + (12918 + i);

View Full Code Here

   * ZkHelixManager.disconnect()->reset() TODO: Refactor accessing
   * HelixMaangerMain class statically
   */
  @Override
  public synchronized void onControllerChange(NotificationContext changeContext) {
    HelixManager manager = changeContext.getManager();
    if (manager == null) {
      LOG.error("missing attributes in changeContext. requires HelixManager");
      return;
    }


    InstanceType type = manager.getInstanceType();
    if (type != InstanceType.CONTROLLER && type != InstanceType.CONTROLLER_PARTICIPANT) {
      LOG.error("fail to become controller because incorrect instanceType (was " + type.toString()
          + ", requires CONTROLLER | CONTROLLER_PARTICIPANT)");
      return;
    }


    try {
      if (changeContext.getType().equals(NotificationContext.Type.INIT)
          || changeContext.getType().equals(NotificationContext.Type.CALLBACK)) {
        // DataAccessor dataAccessor = manager.getDataAccessor();
        HelixDataAccessor accessor = manager.getHelixDataAccessor();
        Builder keyBuilder = accessor.keyBuilder();


        while (accessor.getProperty(keyBuilder.controllerLeader()) == null) {
          boolean success = tryUpdateController(manager);
          if (success) {
            updateHistory(manager);
            if (type == InstanceType.CONTROLLER) {
              HelixControllerMain.addListenersToController(manager, _controller);
              manager.startTimerTasks();
            } else if (type == InstanceType.CONTROLLER_PARTICIPANT) {
              String clusterName = manager.getClusterName();
              String controllerName = manager.getInstanceName();
              _leader =
                  HelixManagerFactory.getZKHelixManager(clusterName, controllerName,
                      InstanceType.CONTROLLER, _zkAddr);


              _leader.connect();

View Full Code Here

  }


  @Test()
  public void TestSchedulerMsg2() throws Exception {
    _factory._results.clear();
    HelixManager manager = null;
    for (int i = 0; i < NODE_NR; i++) {
      _participants[i].getMessagingService().registerMessageHandlerFactory(
          _factory.getMessageType(), _factory);


      manager = _participants[i]; // _startCMResultMap.get(hostDest)._manager;
    }


    Message schedulerMessage =
        new Message(MessageType.SCHEDULER_MSG + "", UUID.randomUUID().toString());
    schedulerMessage.setTgtSessionId("*");
    schedulerMessage.setTgtName("CONTROLLER");
    // TODO: change it to "ADMIN" ?
    schedulerMessage.setSrcName("CONTROLLER");


    // Template for the individual message sent to each participant
    Message msg = new Message(_factory.getMessageType(), "Template");
    msg.setTgtSessionId("*");
    msg.setMsgState(MessageState.NEW);


    // Criteria to send individual messages
    Criteria cr = new Criteria();
    cr.setInstanceName("localhost_%");
    cr.setRecipientInstanceType(InstanceType.PARTICIPANT);
    cr.setSessionSpecific(false);
    cr.setResource("%");
    cr.setPartition("%");


    ObjectMapper mapper = new ObjectMapper();
    SerializationConfig serializationConfig = mapper.getSerializationConfig();
    serializationConfig.set(SerializationConfig.Feature.INDENT_OUTPUT, true);


    StringWriter sw = new StringWriter();
    mapper.writeValue(sw, cr);


    String crString = sw.toString();


    schedulerMessage.getRecord().setSimpleField("Criteria", crString);
    schedulerMessage.getRecord().setMapField("MessageTemplate", msg.getRecord().getSimpleFields());
    schedulerMessage.getRecord().setSimpleField("TIMEOUT", "-1");
    schedulerMessage.getRecord().setSimpleField("WAIT_ALL", "true");


    Criteria cr2 = new Criteria();
    cr2.setRecipientInstanceType(InstanceType.CONTROLLER);
    cr2.setInstanceName("*");
    cr2.setSessionSpecific(false);


    schedulerMessage.getRecord().setSimpleField(
        DefaultSchedulerMessageHandlerFactory.SCHEDULER_TASK_QUEUE, "TestSchedulerMsg2");
    MockAsyncCallback callback = new MockAsyncCallback();
    manager.getMessagingService().sendAndWait(cr2, schedulerMessage, callback, -1);
    String msgId =
        callback._message.getResultMap()
            .get(DefaultSchedulerMessageHandlerFactory.SCHEDULER_MSG_ID);


    HelixDataAccessor helixDataAccessor = manager.getHelixDataAccessor();
    Builder keyBuilder = helixDataAccessor.keyBuilder();
    for (int i = 0; i < 10; i++) {
      Thread.sleep(200);
      PropertyKey controllerTaskStatus =
          keyBuilder.controllerTaskStatus(MessageType.SCHEDULER_MSG.toString(), msgId);

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.helix.HelixManager

com.linkedin.databus.client.pub.ClusterCheckpointPersistenceProvider

com.linkedin.databus.client.pub.ClusterCheckpointPersistenceProvider$HelixConnectionManager

org.apache.helix.agent.AgentStateModel

org.apache.helix.agent.HelixAgentMain

org.apache.helix.controller.GenericHelixController

org.apache.helix.controller.HelixControllerMain

org.apache.helix.controller.stages.BestPossibleStateCalcStage

org.apache.helix.controller.stages.CompatibilityCheckStage

org.apache.helix.controller.stages.ContainerProvisioningStage

org.apache.helix.controller.stages.ExternalViewComputeStage

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.