Package redis.clients.jedis

Examples of redis.clients.jedis.ShardedJedis


      String messageId = getMemQueueMessageMessageId(queueUrlHash, receiptHandle);
      int shard = Util.getShardFromReceiptHandle(receiptHandle);
        boolean cacheAvailable = checkCacheConsistency(queueUrl, shard, false);
       
        if (cacheAvailable) {
            ShardedJedis jedis = null;
            boolean brokenJedis = false;
            try {
                jedis = getResource();
                long ts1 = System.currentTimeMillis();
                long numDeleted = jedis.hdel(queueUrl + "-" + shard + "-H", receiptHandle);
                if (numDeleted != 1) {
                    logger.warn("event=delete_message error_code=could_not_delelete_hidden_set queue_url=" + queueUrl + " shard=" + shard + " mem_id=" + receiptHandle);
                }
                if (jedis.del(queueUrl + "-" + shard + "-A-" + receiptHandle) == 0) {
                    logger.warn("event=delete_message error_code=could_not_delete_attributes queue_url=" + queueUrl + " shard=" + shard + " mem_id=" + receiptHandle);
                }
                long ts2 = System.currentTimeMillis();
                CQSControllerServlet.valueAccumulator.addToCounter(AccumulatorName.RedisTime, (ts2 - ts1));
            } catch (JedisConnectionException e) {
View Full Code Here


       
        if (cacheAvailable) {   
         
          // get the messageIds from in redis list
           
          ShardedJedis jedis = null;
            boolean brokenJedis = false;
           
            try {
               
              jedis = getResource();

                // process revisible-set before getting from queue
               
                tryCheckAndProcessRevisibleSet(queue.getRelativeUrl(), shard, CMBProperties.getInstance().getRedisRevisibleSetFrequencySec());

                try {
                    if (checkAndSetVisibilityProcessing(queue.getRelativeUrl(), shard, CMBProperties.getInstance().getRedisRevisibleFrequencySec())) {                                               
                        logger.debug("event=scheduled_revisibility_processor");
                    }
                } catch (SetFailedException e) {
                }

                boolean done = false;
               
                while (!done) {

                  boolean emptyQueue = false;
                    HashMap<String, String> messageIdToMemId = new HashMap<String, String>();
                    List<String> messageIds = new LinkedList<String>();
                   
                    for (int i = 0; i < maxNumberOfMessages && !emptyQueue; i++) {
                   
                      long ts1 = System.currentTimeMillis();
                        String memId;
                       
                        if (visibilityTO > 0) {
                            memId = jedis.lpop(queue.getRelativeUrl() + "-" + shard + "-Q");
                        } else {
                            memId = jedis.lindex(queue.getRelativeUrl() + "-" + shard + "-Q", i);
                        }
                       
                        long ts2 = System.currentTimeMillis();
                        CQSControllerServlet.valueAccumulator.addToCounter(AccumulatorName.RedisTime, (ts2 - ts1));
                       
                        if (memId == null || memId.equals("nil")) { //done
                            emptyQueue = true;
                        } else {
                            String messageId = getMemQueueMessageMessageId(queue.getRelativeUrlHash(),memId);
                            messageIds.add(messageId);
                            messageIdToMemId.put(messageId, memId);
                        }
                    }
                   
                    if (messageIds.size() == 0) {
                        CQSMonitor.getInstance().registerEmptyResp(queue.getRelativeUrl(), 1);
                        return Collections.emptyList();
                    }
                   
                    // By here messageIds have Underlying layer's messageids.
                    // Get messages from underlying layer. The total returned may not be what was
                    // in mem cache since master-master replication to Cassandra could mean other
                    // colo deleted the message form underlying storage.

                    logger.debug("event=found_msg_ids_in_redis num_mem_ids=" + messageIds.size());
                   
                    try {
                       
                      Map<String, CQSMessage> persisMap = persistenceStorage.getMessages(queue.getRelativeUrl(), messageIds);
                       
                      for (Entry<String, CQSMessage> messageIdToMessage : persisMap.entrySet()) {
                       
                        String memId = messageIdToMemId.get(messageIdToMessage.getKey());
                           
                        if (memId == null) {
                              throw new IllegalStateException("Underlying storage layer returned a message that was not requested");
                            }
                           
                        CQSMessage message = messageIdToMessage.getValue();
                           
                        if (message == null) {
                              logger.warn("event=message_is_null msg_id=" + messageIdToMessage.getKey());
                              continue;
                            }
                           
                            //check if message returned is expired. This will only happen if payload coming from payload-cache. Cassandra
                            //expires old messages just fine. If expired, skip over and continue.
                           
                        if (isMessageExpired(queue, memId)) {
                           
                          try {
                                    logger.debug("event=message_expired message_id=" + memId);
                                  persistenceStorage.deleteMessage(queue.getRelativeUrl(), message.getMessageId());
                                } catch (Exception e) {
                                    //its fine to ignore this exception since message must have been auto-deleted from Cassandra
                                    //but we need to do this to clean up payloadcache
                                    logger.debug("event=message_already_deleted_in_cassandra msg_id=" + message.getMessageId());
                                }
                               
                          continue;
                            }

                            //hide message if visibilityTO is greater than 0
                           
                        if (visibilityTO > 0) {
                                long ts1 = System.currentTimeMillis();
                                jedis.hset(queue.getRelativeUrl() + "-" + shard + "-H", memId, Long.toString(System.currentTimeMillis() + (visibilityTO * 1000)));
                                long ts2 = System.currentTimeMillis();
                                CQSControllerServlet.valueAccumulator.addToCounter(AccumulatorName.RedisTime, (ts2 - ts1));
                            }

                            message.setMessageId(memId);
                            message.setReceiptHandle(memId);

                            //get message-attributes and populate in message
                            Map<String, String> msgAttrs = (message.getAttributes() != null) ?  message.getAttributes() : new HashMap<String, String>();
                            List<String> attrs = jedis.hmget(queue.getRelativeUrl() + "-" + shard + "-A-" + memId, CQSConstants.REDIS_APPROXIMATE_FIRST_RECEIVE_TIMESTAMP, CQSConstants.REDIS_APPROXIMATE_RECEIVE_COUNT);
                           
                            if (attrs.get(0) == null) {
                                String firstRecvTS = Long.toString(System.currentTimeMillis());
                                jedis.hset(queue.getRelativeUrl() + "-" + shard + "-A-" + memId, CQSConstants.REDIS_APPROXIMATE_FIRST_RECEIVE_TIMESTAMP, firstRecvTS);
                                msgAttrs.put(CQSConstants.APPROXIMATE_FIRST_RECEIVE_TIMESTAMP, firstRecvTS);
                            } else {                           
                                msgAttrs.put(CQSConstants.APPROXIMATE_FIRST_RECEIVE_TIMESTAMP, attrs.get(0));
                            }
                           
                            int recvCount = 1;
                           
                            if (attrs.get(1) != null) {
                                recvCount = Integer.parseInt(attrs.get(1)) + 1;
                            }
                           
                            jedis.hset(queue.getRelativeUrl() + "-" + shard + "-A-" + memId, CQSConstants.REDIS_APPROXIMATE_RECEIVE_COUNT, Integer.toString(recvCount));
                            jedis.expire(queue.getRelativeUrl() + "-" + shard +  "-A-" + memId, 3600 * 24 * 14); //14 days expiration if not deleted
                            msgAttrs.put(CQSConstants.APPROXIMATE_RECEIVE_COUNT, Integer.toString(recvCount));
                            message.setAttributes(msgAttrs);
                            ret.add(message);
                        }
                       
                        if (ret.size() > 0) { //There may be cases where the underlying persistent message has two memIds while
                            //cache filling. In such cases trying to retrieve message for the second memId may result in no message
                            //returned. We should skip over those memIds and continue till we find at least one valid memId
                            done = true;
                        } else {
                            for (String messageId : messageIds) {
                                logger.debug("event=bad_mem_id_found msg_id=" + messageId + " action=skip_message");
                            }
                        }
                       
                    } catch (HTimedOutException e1) { //If Hector timedout, push messages back
                        logger.error("event=hector_timeout num_messages=" + messageIds.size() + " action=pushing_messages_back_to_redis");
                        if (visibilityTO > 0) {
                            for (String messageId : messageIds) {
                                String memId = messageIdToMemId.get(messageId);
                                jedis.lpush(queue.getRelativeUrl() + "-" + shard + "-Q", memId);
                            }
                        }
                        throw e1;
                    }
                }
View Full Code Here

      int shard = Util.getShardFromReceiptHandle(receiptHandle);
      boolean cacheAvailable = checkCacheConsistency(queue.getRelativeUrl(), shard, false);

      if (cacheAvailable) {
       
            ShardedJedis jedis = null;
            boolean brokenJedis = false;
           
            try {
             
                jedis = getResource();
                long ts1 = System.currentTimeMillis();
                if (visibilityTO == 0) { //make immediately visible
                    jedis.rpush(queue.getRelativeUrl() + "-" + shard + "-Q", receiptHandle);
                   jedis.hdel(queue.getRelativeUrl() + "-" + shard + "-H", receiptHandle);
                    return true;
                } else { //update new visibilityTO
                  jedis.hset(queue.getRelativeUrl() + "-" + shard + "-H", receiptHandle, Long.toString(System.currentTimeMillis() + (visibilityTO * 1000)));                 
                }
               
                long ts2 = System.currentTimeMillis();
                CQSControllerServlet.valueAccumulator.addToCounter(AccumulatorName.RedisTime, (ts2 - ts1));
                return true;
View Full Code Here

        List<String> memIdsRet = getIdsFromHead(queueUrl, shard, previousReceiptHandle, length);               
            // by here memIdsRet should have memIds to return messages for
            Map<String, CQSMessage> ret = getMessages(queueUrl, memIdsRet);
            // return list in the same order as memIdsRet
            List<CQSMessage> messages = new LinkedList<CQSMessage>();
            ShardedJedis jedis = null;
            boolean brokenJedis = false;
           
            try {
           
              jedis = getResource();
             
              for (String memId : memIdsRet) {
             
                CQSMessage message = ret.get(memId);
                 
                if (message != null) {
                 
                    // get message-attributes and populate in message
                 
                      Map<String, String> msgAttrs = (message.getAttributes() != null) ?  message.getAttributes() : new HashMap<String, String>();
                      List<String> attrs = jedis.hmget(queueUrl + "-" + shard + "-A-" + memId, CQSConstants.REDIS_APPROXIMATE_FIRST_RECEIVE_TIMESTAMP, CQSConstants.REDIS_APPROXIMATE_RECEIVE_COUNT);
                     
                      if (attrs.get(0) != null) {
                          msgAttrs.put(CQSConstants.APPROXIMATE_FIRST_RECEIVE_TIMESTAMP, attrs.get(0));
                      }
                     
View Full Code Here

      boolean cacheAvailable = checkCacheConsistency(queueUrl, shard, true);
       
      if (cacheAvailable) {
           
        boolean brokenJedis = false;
            ShardedJedis jedis = null;
           
            try {
                long ts1 = System.currentTimeMillis();
                jedis = getResource();
                Long num = jedis.del(queueUrl + "-" + shard + "-" + CQSConstants.REDIS_STATE);
                logger.debug("num removed=" + num);
                num = jedis.del(queueUrl + "-" + shard + "-Q");
                logger.debug("num removed=" + num);
                num = jedis.del(queueUrl + "-" + shard + "-H");
                logger.debug("num removed=" + num);
                num = jedis.del(queueUrl + "-" + shard + "-R");
                logger.debug("num removed=" + num);
                num = jedis.del(queueUrl + "-" + shard + "-F");
                logger.debug("num removed=" + num);
                num = jedis.del(queueUrl + "-" + shard + "-V");
                logger.debug("num removed=" + num);
                num = jedis.del(queueUrl + "-" + shard + "-VR");
                logger.debug("num removed=" + num);
                long ts2 = System.currentTimeMillis();
                CQSControllerServlet.valueAccumulator.addToCounter(AccumulatorName.RedisTime, (ts2 - ts1));
                logger.debug("event=cleared_queue queue_url=" + queueUrl + " shard=" + shard);
            } catch (JedisConnectionException e) {
View Full Code Here

          return Collections.emptyList();
        }
       
        List<String> memIdsRet = new LinkedList<String>();
        boolean brokenJedis = false;
        ShardedJedis jedis = null;
       
        try {
       
          jedis = getResource();
           
            // process re-visible set before getting from queue
           
            tryCheckAndProcessRevisibleSet(queueUrl, shard, CMBProperties.getInstance().getRedisRevisibleSetFrequencySec());

            try {
                if (checkAndSetVisibilityProcessing(queueUrl, shard, CMBProperties.getInstance().getRedisRevisibleFrequencySec())) {                                               
                    logger.debug("event=scheduled_revisibility_processor");
                }
            } catch (SetFailedException e) {
            }

            long llen = jedis.llen(queueUrl + "-" + shard + "-Q");
           
            if (llen == 0L) {
                return Collections.emptyList();
            }
           
            int retCount = 0;
            long i = 0L;
            boolean includeSet = (previousReceiptHandle == null) ? true : false;
           
            while (retCount < num && i < llen) {
           
              List<String> memIds = jedis.lrange(queueUrl + "-" + shard + "-Q", i, i + num - 1);
               
              if (memIds.size() == 0) {
                  break; // done
                }
               
View Full Code Here

     
      if (queue != null) {
        numberOfShards = queue.getNumberOfShards();
      }

    ShardedJedis jedis = null;
        boolean brokenJedis = false;
       
        try {

          jedis = getResource();
           
          for (int shard=0; shard<numberOfShards; shard++) {
           
            boolean cacheAvailable = checkCacheConsistency(queueUrl, shard, true);
               
                if (!cacheAvailable) {
                  throw new IllegalStateException("Redis cache not available");
                }
               
                if (processHiddenIds) {
                  checkAndSetVisibilityProcessing(queueUrl, shard, CMBProperties.getInstance().getRedisRevisibleFrequencySec());
                    tryCheckAndProcessRevisibleSet(queueUrl, shard, CMBProperties.getInstance().getRedisRevisibleSetFrequencySec());
                }
             
                messageCount += jedis.llen(queueUrl + "-" + shard + "-Q");
            }
         
        } catch (JedisException e) {
            brokenJedis = true;
            throw e;
View Full Code Here

     
      if (queue != null) {
        numberOfShards = queue.getNumberOfShards();
      }

    ShardedJedis jedis = null;
        boolean brokenJedis = false;
       
        try {

          jedis = getResource();
           
          for (int shard=0; shard<numberOfShards; shard++) {          
                messageCount += jedis.llen(queueUrl + "-" + shard + "-Q");
            }
         
        } catch (JedisException e) {
            brokenJedis = true;
            throw e;
View Full Code Here

     
      if (queue != null) {
        numberOfShards = queue.getNumberOfShards();
      }

    ShardedJedis jedis = null;
        boolean brokenJedis = false;
       
        try {

          jedis = getResource();
           
          for (int shard=0; shard<numberOfShards; shard++) {
           
            boolean cacheAvailable = checkCacheConsistency(queueUrl, shard, true);
               
                if (!cacheAvailable) {
                  throw new IllegalStateException("Redis cache not available");
                }
                //if check for hidden message, and processFlag is true, move the message from H to Q
                if(suffix.equals("-H")&&(processHiddenIds)){
                  checkAndSetVisibilityProcessing(queueUrl, shard, CMBProperties.getInstance().getRedisRevisibleFrequencySec());                                                
                }
                //if check for delayed message, and processFlag is true, move the message from V to Q                 
                if (suffix.equals("-V")&&(processHiddenIds)) {
                    tryCheckAndProcessRevisibleSet(queueUrl, shard, CMBProperties.getInstance().getRedisRevisibleSetFrequencySec());
                }
             
                //get the count number from
                if(suffix.equals("-H")){
                  messageCount += jedis.hlen(queueUrl + "-" + shard + suffix);
                } else if (suffix.equals("-V")){
                  messageCount += jedis.zcard(queueUrl + "-" + shard + suffix);
                }
            }
         
        } catch (JedisException e) {
            brokenJedis = true;
View Full Code Here

   * @return list of hash maps, one for each shard
   */
  public List<Map<String, String>> getInfo() {
     
    boolean brokenJedis = false;
      ShardedJedis jedis = getResource();
      List<Map<String, String>> shardInfos = new ArrayList<Map<String, String>>();
     
      try {
     
        Collection<Jedis> shards = jedis.getAllShards();
 
          for (Jedis shard : shards) {
         
            String info = shard.info();
          String lines[] = info.split("\n");
View Full Code Here

TOP

Related Classes of redis.clients.jedis.ShardedJedis

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.