Package com.comcast.cqs.model

Examples of com.comcast.cqs.model.CQSMessage


         
          ReceiveMessageResult receiveMessageResult = sqs.receiveMessage(receiveMessageRequest);
          List<Message> msgs = receiveMessageResult.getMessages();
         
          for (Message m : msgs) {
            messages.add(new CQSMessage(m));
          }
        }

        long ts2 = System.currentTimeMillis();
        CMBControllerServlet.valueAccumulator.addToCounter(AccumulatorName.CNSCQSTime, ts2 - ts1);
View Full Code Here


            }
         
          String publishJobQName = CNS_PRODUCER_QUEUE_NAME_PREFIX + partition;
          String queueUrl = CQSHandler.getRelativeCnsInternalQueueUrl(publishJobQName);
         
          CQSMessage msg = null;
          int waitTimeSecs = 0;
         
          if (CMBProperties.getInstance().isCQSLongPollEnabled()) {
            waitTimeSecs = CMBProperties.getInstance().getCMBRequestTimeoutSec();
          }

          List<CQSMessage> l = CQSHandler.receiveMessage(queueUrl, waitTimeSecs, 1);

          if (l.size() > 0) {
            msg = l.get(0);
          }

          CNSWorkerMonitor.getInstance().registerCQSServiceAvailable(true);

          if (msg != null) {
           
            // if long polling disabled and message found reset exponential backoff
           
            if (!CMBProperties.getInstance().isCQSLongPollEnabled()) {
              processingDelayMillis = 10;
            }
           
              CNSMessage publishMessage = CNSMessage.parseInstance(msg.getBody());
             
              int messageExpirationSeconds = CMBProperties.getInstance().getCNSMessageExpirationSeconds();
             
              if (messageExpirationSeconds != 0 && System.currentTimeMillis() - publishMessage.getTimestamp().getTime() > messageExpirationSeconds*1000) {
                  logger.error("event=deleting_publish_job reason=message_too_old topic_arn=" + publishMessage.getTopicArn());
                  CQSHandler.deleteMessage(queueUrl, msg.getReceiptHandle());
                  CMBControllerServlet.valueAccumulator.deleteAllCounters();
                  return true; // return true to avoid backoff
              }
             
              List<CNSEndpointPublishJob.CNSEndpointSubscriptionInfo> subscriptions = null;
              long t1=System.currentTimeMillis();
              try {
                  subscriptions = getSubscriptionsForTopic(publishMessage.getTopicArn());
              } catch (TopicNotFoundException e) {

                //delete this message/job since the topic was deleted.
                 
                logger.error("event=deleting_publish_job reason=topic_not_found topic_arn=" + publishMessage.getTopicArn());
                  CQSHandler.deleteMessage(queueUrl, msg.getReceiptHandle());
                  CMBControllerServlet.valueAccumulator.deleteAllCounters();
                  return true; // return true to avoid backoff
             
              } catch (Exception ex) {
                  logger.error("event=skipping_publish_job reason=error_fetching_subscriptions", ex);
                  CMBControllerServlet.valueAccumulator.deleteAllCounters();
                  return true; // return true to avoid backoff
              }
              logger.debug("event=get_subscription_list ms="+(System.currentTimeMillis()-t1));
              if (subscriptions != null && subscriptions.size() > 0) {
               
                  messageFound = true;
                  List<CNSEndpointPublishJob> epPublishJobs = createEndpointPublishJobs(publishMessage, subscriptions);
                 
                  for (CNSEndpointPublishJob epPublishJob: epPublishJobs) {
                   
                    String epQueueName =  CMBProperties.getInstance().getCNSEndpointPublishQueueNamePrefix() + ((new Random()).nextInt(CMBProperties.getInstance().getCNSNumEndpointPublishJobQueues()));
                      String epQueueUrl = CQSHandler.getRelativeCnsInternalQueueUrl(epQueueName);
                      CQSHandler.sendMessage(epQueueUrl, epPublishJob.serialize());
                  }
              }
             
              CQSHandler.deleteMessage(queueUrl, msg.getReceiptHandle());
             
              long ts2 = System.currentTimeMillis();
              logger.debug("event=processed_producer_job cns_cqs_ms=" + CMBControllerServlet.valueAccumulator.getCounter(AccumulatorName.CNSCQSTime) + " resp_ms=" + (ts2 - ts1));

          } else {
View Full Code Here

                return messageFound;
            }
         
          String queueName = CNS_CONSUMER_QUEUE_NAME_PREFIX + partition;
          String queueUrl = CQSHandler.getRelativeCnsInternalQueueUrl(queueName);
          CQSMessage msg = null;
          int waitTimeSecs = 0;
         
          if (CMBProperties.getInstance().isCQSLongPollEnabled()) {
            waitTimeSecs = CMBProperties.getInstance().getCMBRequestTimeoutSec();
          }

          List<CQSMessage> l = CQSHandler.receiveMessage(queueUrl, waitTimeSecs, 1);

          if (l.size() > 0) {
            msg = l.get(0);
          }
         
        CNSWorkerMonitor.getInstance().registerCQSServiceAvailable(true);

            if (msg != null) {  
             
            // if long polling disabled and message found reset exponential backoff
           
            if (!CMBProperties.getInstance().isCQSLongPollEnabled()) {
              processingDelayMillis = 10;
            }
             
                messageFound = true;
               
                try {
                 
                    CNSEndpointPublishJob endpointPublishJob = (CMBProperties.getInstance().isCNSUseSubInfoCache()) ? CNSCachedEndpointPublishJob.parseInstance(msg.getBody()) : CNSEndpointPublishJob.parseInstance(msg.getBody());
                    logger.debug("endpoint_publish_job=" + endpointPublishJob.toString());
                    User pubUser =(new UserAuthModule()).getUserByUserId(endpointPublishJob.getMessage().getUserId());
                    List<? extends CNSEndpointSubscriptionInfo> subs = endpointPublishJob.getSubInfos();
                   
                    CNSWorkerMonitor.getInstance().registerSendsRemaining(endpointPublishJob.getMessage().getMessageId(), subs.size());
                   
                    AtomicInteger endpointPublishJobCount = new AtomicInteger(subs.size());               
                   
                    for (CNSEndpointSubscriptionInfo sub : subs) {            
                       
                      Runnable publishJob = null;
                      CNSMessage message = endpointPublishJob.getMessage();
                      message.setSubscriptionArn(sub.subArn);
                       
                        if (CMBProperties.getInstance().getCNSIOMode() == IO_MODE.SYNC) {
                          publishJob = new CNSPublishJob(message, pubUser, sub.protocol, sub.endpoint, sub.subArn, sub.rawDelivery, queueUrl, msg.getReceiptHandle(), endpointPublishJobCount);
                        } else {
                          publishJob = new CNSAsyncPublishJob(message, pubUser, sub.protocol, sub.endpoint, sub.subArn, sub.rawDelivery, queueUrl, msg.getReceiptHandle(), endpointPublishJobCount);
                        }

                        deliveryHandlers.submit(publishJob);
                    }
                   
                } catch (TopicNotFoundException e) {
                    logger.error("event=topic_not_found action=skip_job");
                    CQSHandler.deleteMessage(queueUrl, msg.getReceiptHandle());
                } catch (Exception e) {
                    logger.error("event=job_consumer_exception action=wait_for_revisibility", e);
                }
               
                long tsFinal = System.currentTimeMillis();
View Full Code Here

               if (memId == null) {
                 throw new IllegalStateException("Underlying storage layer returned a message that was not requested");
               }

               CQSMessage message = messageIdToMessage.getValue();

               if (message == null) {
                 logger.warn("event=message_is_null msg_id=" + messageIdToMessage.getKey());
                 //underlying layer does not have this message, remove it from Redis layer
                 jedis.zrem(key,memId);
                 continue;
               }

               message.setMessageId(memId);
               message.setReceiptHandle(memId);

               //get message-attributes and populate in message
               Map<String, String> msgAttrs = (message.getAttributes() != null) ?  message.getAttributes() : new HashMap<String, String>();
               List<String> attrs = jedis.hmget(queue.getRelativeUrl() + "-" + shard + "-A-" + memId, CQSConstants.REDIS_APPROXIMATE_FIRST_RECEIVE_TIMESTAMP, CQSConstants.REDIS_APPROXIMATE_RECEIVE_COUNT);

               if (attrs.get(0) == null) {
                 String firstRecvTS = Long.toString(System.currentTimeMillis());
                 jedis.hset(queue.getRelativeUrl() + "-" + shard + "-A-" + memId, CQSConstants.REDIS_APPROXIMATE_FIRST_RECEIVE_TIMESTAMP, firstRecvTS);
                 msgAttrs.put(CQSConstants.APPROXIMATE_FIRST_RECEIVE_TIMESTAMP, firstRecvTS);
               } else {                           
                 msgAttrs.put(CQSConstants.APPROXIMATE_FIRST_RECEIVE_TIMESTAMP, attrs.get(0));
               }

               int recvCount = 1;

               if (attrs.get(1) != null) {
                 recvCount = Integer.parseInt(attrs.get(1)) + 1;
               }

               jedis.hset(queue.getRelativeUrl() + "-" + shard + "-A-" + memId, CQSConstants.REDIS_APPROXIMATE_RECEIVE_COUNT, Integer.toString(recvCount));
               jedis.expire(queue.getRelativeUrl() + "-" + shard +  "-A-" + memId, 3600 * 24 * 14); //14 days expiration if not deleted
               msgAttrs.put(CQSConstants.APPROXIMATE_RECEIVE_COUNT, Integer.toString(recvCount));
               message.setAttributes(msgAttrs);
               ret.add(message);
             }

             if (ret.size() > 0) { //There may be cases where the underlying persistent message has two memIds while
               //cache filling. In such cases trying to retrieve message for the second memId may result in no message
View Full Code Here

         jedis = getResource();

         for (String memId : memIdsRet) {

           CQSMessage message = ret.get(memId);

           if (message != null) {

             // get message-attributes and populate in message

             Map<String, String> msgAttrs = (message.getAttributes() != null) ?  message.getAttributes() : new HashMap<String, String>();
             List<String> attrs = jedis.hmget(queueUrl + "-" + shard + "-A-" + memId, CQSConstants.REDIS_APPROXIMATE_FIRST_RECEIVE_TIMESTAMP, CQSConstants.REDIS_APPROXIMATE_RECEIVE_COUNT);

             if (attrs.get(0) != null) {
               msgAttrs.put(CQSConstants.APPROXIMATE_FIRST_RECEIVE_TIMESTAMP, attrs.get(0));
             }

             if (attrs.get(1) != null) {
               msgAttrs.put(CQSConstants.APPROXIMATE_RECEIVE_COUNT, Integer.toString(Integer.parseInt(attrs.get(1))));
             }

             message.setAttributes(msgAttrs);
             messages.add(message);
           }
         }
       } catch (JedisConnectionException e) {
         logger.warn("event=peek_message error_code=redis_unavailable num_connections=" + numRedisConnections.get());
         brokenJedis = true;
         cacheAvailable = false;
       } finally {
         if (jedis != null) {
           returnResource(jedis, brokenJedis);
         }
       }

       logger.debug("event=peek_queue cache=available queue_url=" + queueUrl + " shard=" + shard + " num_messages=" + messages.size());
       return messages;

     } else { //get from underlying storage
       String relativeUrlHash = Util.getQueueUrlHashFromCache(queueUrl);
       if (previousReceiptHandle != null) { //strip out this layer's id
         previousReceiptHandle = getMemQueueMessageMessageId(relativeUrlHash, previousReceiptHandle);
       }

       if (nextReceiptHandle != null) {
         nextReceiptHandle = getMemQueueMessageMessageId(relativeUrlHash, nextReceiptHandle);
       }       

       List<CQSMessage> messages = persistenceStorage.peekQueue(queueUrl, shard, previousReceiptHandle, nextReceiptHandle, length);

       for (CQSMessage message : messages) {
         String memId = getMemQueueMessage(message.getMessageId()); //TODO: initialDelay is 0
         message.setMessageId(memId);
         message.setReceiptHandle(memId);
       }

       logger.debug("event=peek_queue cache=unavailable queue_url=" + queueUrl + " shard=" + shard + " num_messages=" + messages.size());
       return messages;
     }
View Full Code Here

     Map<String, CQSMessage> persisMap = persistenceStorage.getMessages(queueUrl, messageIds);
     Map<String, CQSMessage> ret = new HashMap<String, CQSMessage>();

     for (Entry<String, CQSMessage> idToMessage : persisMap.entrySet()) {
       CQSMessage msg = idToMessage.getValue();
       if (msg == null) {
         continue;
       }
       String memId = messageIdToMemId.get(idToMessage.getKey());
       msg.setMessageId(memId);
       msg.setReceiptHandle(memId);
       ret.put(memId, msg);
     }

     return ret;
   }
View Full Code Here

        attributes.put(CQSConstants.SENDER_ID, user.getUserId());
        attributes.put(CQSConstants.SENT_TIMESTAMP, "" + System.currentTimeMillis());
        attributes.put(CQSConstants.APPROXIMATE_RECEIVE_COUNT, "0");
        attributes.put(CQSConstants.APPROXIMATE_FIRST_RECEIVE_TIMESTAMP, "");
       
        CQSMessage message = new CQSMessage(messageBody, attributes);
      CQSQueue queue = CQSCache.getCachedQueue(user, request);
     
      int shard = 0;
     
    String shardNumber = request.getParameter("Shard");

    if (shardNumber != null) {
      shard = Integer.parseInt(shardNumber);
    } else if (queue.getNumberOfShards() > 1) {
      shard = rand.nextInt(queue.getNumberOfShards());
    }
 
    if (shard < 0 || shard >= queue.getNumberOfShards()) {
          throw new CMBException(CMBErrorCodes.InvalidParameterValue, "Shard number " + shard +" exceeds number of available shards in queue (" + queue.getNumberOfShards() + ").");
    }

        String receiptHandle = PersistenceFactory.getCQSMessagePersistence().sendMessage(queue, shard, message);

        if (receiptHandle == null || receiptHandle.isEmpty()) {
            throw new CMBException(CMBErrorCodes.InternalError, "Failed to add message to queue");
        }
       
        request.setReceiptHandles(Arrays.asList(new String[] {receiptHandle}));
        message.setReceiptHandle(receiptHandle);
        message.setMessageId(receiptHandle);
        String out = CQSMessagePopulator.getSendMessageResponse(message);
        writeResponse(out, response);
        CQSMonitor.getInstance().addNumberOfMessagesReceived(queue.getRelativeUrl(), 1);
       
        try {
View Full Code Here

          continue;
        } else if (column.getValue() == null || column.getValue().length() == 0) {
          continue;
        }
       
        CQSMessage message = extractMessageFromJSON(queueUrl, column);
        messageList.add(message);
      }
     
      if (noMatch && messageList.size() > length) {
        messageList.remove(messageList.size() - 1);
View Full Code Here

  }
 
  private CQSMessage extractMessageFromJSON(String queueUrl, CmbColumn column) throws JSONException, PersistenceException, IOException {
   
    CQSQueue queue = null;
    CQSMessage m = new CQSMessage();
   
    try {
      queue = CQSCache.getCachedQueue(queueUrl);
    } catch (Exception ex) {
      throw new PersistenceException(ex);
    }
   
    if (queue == null) {
      throw new PersistenceException(CMBErrorCodes.InternalError, "Unknown queue " + queueUrl);
    }
   
    JSONObject json = new JSONObject((String)column.getValue());

    m.setMessageId(json.getString("MessageId"));
    m.setReceiptHandle(json.getString("MessageId"));
    m.setMD5OfBody(json.getString("MD5OfBody"));
    m.setBody(json.getString("Body"));
   
    if (m.getAttributes() == null) {
      m.setAttributes(new HashMap<String, String>());
    }
   
    if (json.has(CQSConstants.SENT_TIMESTAMP)) {
      m.getAttributes().put(CQSConstants.SENT_TIMESTAMP, json.getString(CQSConstants.SENT_TIMESTAMP));
    }
   
    if (json.has(CQSConstants.APPROXIMATE_RECEIVE_COUNT)) {
      m.getAttributes().put(CQSConstants.APPROXIMATE_RECEIVE_COUNT, json.getString(CQSConstants.APPROXIMATE_RECEIVE_COUNT));
    }

    if (json.has(CQSConstants.SENDER_ID)) {
      m.getAttributes().put(CQSConstants.SENDER_ID, json.getString(CQSConstants.SENDER_ID));
    }
   
    m.setTimebasedId(column.getName());
   
    if (queue.isCompressed()) {
      m.setBody(Util.decompress(m.getBody()));
    }
   
      return m;
  }
View Full Code Here

    }

  @Test
  public void testSendMessage() throws NoSuchAlgorithmException, PersistenceException, IOException, InterruptedException, JSONException {
   
        CQSMessage message = new CQSMessage("This is a test message " + (new Random()).nextInt(), attributes);
        attributes.put("SentTimestamp", "" + Calendar.getInstance().getTimeInMillis());
    String receiptHandle = persistence.sendMessage(queue, 0, message);
    assertNotNull(receiptHandle);
  }
View Full Code Here

TOP

Related Classes of com.comcast.cqs.model.CQSMessage

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.