Package org.apache.hedwig.protocol.PubSubProtocol

Examples of org.apache.hedwig.protocol.PubSubProtocol.Message

210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
        // start delivery
        OrderCheckingMessageHandler ocm = new OrderCheckingMessageHandler(
                topic, subId, startMsgId, numMsgs);
        subscriber.startDelivery(topic, subId, ocm);
        for (int i=0; i<numMsgs; i++) {
            Message msg = Message.newBuilder().setBody(
                    ByteString.copyFromUtf8(Integer.toString(startMsgId + i))).build();
            publisher.publish(topic, msg);
        }
        logger.info("Publish finished.");
        queue.take();
        logger.info("Deliver finished.");
        // check messages received in order
        assertTrue(ocm.isInOrder());
        // make sure consume request sent to hub server before shut down
        Thread.sleep(2000);
        subscriber.stopDelivery(topic, subId);
        subscriber.closeSubscription(topic, subId);

        stopHubServers();
        Thread.sleep(1000);
        startHubServers();

        startMsgId = 20;
        // reconnect it again
        subscriber.subscribe(topic, subId, CreateOrAttach.CREATE_OR_ATTACH);
        ocm = new OrderCheckingMessageHandler(topic, subId, startMsgId, numMsgs);
        subscriber.startDelivery(topic, subId, ocm);
        for (int i=0; i<numMsgs; i++) {
            Message msg = Message.newBuilder().setBody(
                    ByteString.copyFromUtf8(Integer.toString(startMsgId + i))).build();
            publisher.publish(topic, msg);
        }
        queue.take();
        // check messages received in order
View Full Code Here
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
        Connection conn = threadLocalConnection.get();

        Callback<MessageSeqId> callback = request.getCallback();
        Object ctx = request.getCtx();
        ByteString topic = request.getTopic();
        Message message = request.getMessage();

        if (conn == null) {
            callback.operationFailed(ctx, new ServiceDownException("Not connected to derby"));
            return;
        }

        long seqId;

        try {
            seqId = adjustTopicSeqIdForPublish(topic, message);
        } catch (UnexpectedConditionException e) {
            callback.operationFailed(ctx, e);
            return;
        }
        PreparedStatement stmt;

        boolean triedCreatingTable = false;
        while (true) {
            try {
                message.getBody();
                stmt = conn.prepareStatement("INSERT INTO " + getTableNameForTopic(topic) + " VALUES(?,?)");
                stmt.setLong(1, seqId);
                stmt.setBlob(2, new SerialBlob(message.toByteArray()));

                int rowCount = stmt.executeUpdate();
                stmt.close();
                if (rowCount != 1) {
                    logger.error("Unexpected number of affected rows from derby");
View Full Code Here
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
                    Message.Builder messageBuilder = Message.newBuilder().mergeFrom(resultSet.getBinaryStream(2));

                    // Merge in the local seq-id since that is not stored with
                    // the message
                    Message message = MessageIdUtils.mergeLocalSeqId(messageBuilder, localSeqId);

                    callback.messageScanned(ctx, message);
                    numMessages++;
                    totalSize += message.getBody().size();

                    if (numMessages > messageLimit) {
                        stmt.close();
                        callback.scanFinished(ctx, ReasonForFinish.NUM_MESSAGES_LIMIT_EXCEEDED);
                        return;
View Full Code Here
673
674
675
676
677
678
679
680
681
682
683
684
685
686
                topicInfo.lastSeqIdBeforeLedgerChange = localSeqId;
            }
        }

        topicInfo.lastSeqIdPushed = builder.build();
        Message msgToSerialize = Message.newBuilder(request.message).setMsgId(topicInfo.lastSeqIdPushed).build();

        final MessageSeqId responseSeqId = msgToSerialize.getMsgId();
        topicInfo.currentLedgerRange.handle.asyncAddEntry(msgToSerialize.toByteArray(),
        new SafeAsynBKCallback.AddCallback() {
            AtomicBoolean processed = new AtomicBoolean(false);
            @Override
            public void safeAddComplete(int rc, LedgerHandle lh, long entryId, Object ctx) {
View Full Code Here
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
                    }

                    LedgerEntry entry = null;
                    while (seq.hasMoreElements()) {
                        entry = seq.nextElement();
                        Message message;
                        try {
                            message = Message.parseFrom(entry.getEntryInputStream());
                        } catch (IOException e) {
                            String msg = "Unreadable message found in ledger: " + imlr.range.getLedgerId()
                                         + " for topic: " + topic.toStringUtf8();
                            logger.error(msg, e);
                            request.callback.scanFailed(ctx, new PubSubException.UnexpectedConditionException(msg));
                            return;
                        }

                        logger.debug("Read response from ledger: {} entry-id: {}",
                                     lh.getId(), entry.getEntryId());

                        assert expectedEntryId == entry.getEntryId() : "expectedEntryId (" + expectedEntryId
                        + ") != entry.getEntryId() (" + entry.getEntryId() + ")";
                        assert (message.getMsgId().getLocalComponent() - imlr.getStartSeqIdIncluded()) == expectedEntryId;

                        expectedEntryId++;
                        request.callback.messageScanned(ctx, message);
                        numMessagesRead++;
                        totalSizeRead += message.getBody().size();

                        if (numMessagesRead >= request.messageLimit) {
                            request.callback.scanFinished(ctx, ReasonForFinish.NUM_MESSAGES_LIMIT_EXCEEDED);
                            return;
                        }
View Full Code Here
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
                                             + topic.toStringUtf8() + ", could not read last entry", bke);
                                cb.operationFailed(ctx, new PubSubException.ServiceDownException(bke));
                                return;
                            }

                            Message lastMessage;
                            try {
                                lastMessage = Message.parseFrom(seq.nextElement().getEntry());
                            } catch (InvalidProtocolBufferException e) {
                                String msg = "While recovering ledger: " + ledgerId + " for topic: "
                                             + topic.toStringUtf8() + ", could not deserialize last message";
                                logger.error(msg, e);
                                cb.operationFailed(ctx, new PubSubException.UnexpectedConditionException(msg));
                                return;
                            }

                            long endOfLedger  = lastMessage.getMsgId().getLocalComponent();
                            long startOfLedger = endOfLedger - numEntriesInLastLedger + 1;

                            if (startOfLedger != expectedStartSeqId) {
                                // gap would be introduced by old version when gc consumed ledgers
                                String msg = "Expected start seq id of recovered ledger " + ledgerId
                                             + " to be " + expectedStartSeqId + " but it was "
                                             + startOfLedger + ".";
                                logger.warn(msg);
                            }

                            LedgerRange lr = buildLedgerRange(ledgerId, startOfLedger, lastMessage.getMsgId());
                            topicInfo.ledgerRanges.put(endOfLedger,
                                    new InMemoryLedgerRange(lr, lh));

                            logger.info("Recovered unclosed ledger: {} for topic: {} with {} entries starting from seq id {}",
                                        va(ledgerId, topic.toStringUtf8(), numEntriesInLastLedger, startOfLedger));
View Full Code Here
244
245
246
247
248
249
250
251
252
253
254
            // hear success
            originalRequest.getCallback().operationFinished(originalRequest.getCtx(), resultOfOperation);

            // Original message that was persisted didn't have the local seq-id.
            // Lets add that in
            Message messageWithLocalSeqId = MessageIdUtils.mergeLocalSeqId(originalRequest.getMessage(),
                                            resultOfOperation.getLocalComponent());

            // Now enqueue a request to add this newly persisted message to our
            // cache
            CacheKey cacheKey = new CacheKey(originalRequest.getTopic(), resultOfOperation.getLocalComponent());
View Full Code Here
63
64
65
66
67
68
69
70
71
72
73
        // picking constants arbitarily for warmup phase
        ThroughputLatencyAggregator agg = new ThroughputLatencyAggregator("acked pubs", nWarmup, 100);
        agg.startProgress();

        Message msg = getMsg(1024);
        for (int i = 0; i < nWarmup; i++) {
            publisher.asyncPublish(topic, msg, new BenchmarkCallback(agg), null);
        }

        if (agg.tpAgg.queue.take() > 0) {
View Full Code Here
80
81
82
83
84
85
86
87
        StringBuilder sb = new StringBuilder();
        for (int i = 0; i < size; i++) {
            sb.append('a');
        }
        final ByteString body = ByteString.copyFromUtf8(sb.toString());
        Message msg = Message.newBuilder().setBody(body).build();
        return msg;
    }
View Full Code Here
85
86
87
88
89
90
91
92
93
94
95
        Message msg = Message.newBuilder().setBody(body).build();
        return msg;
    }

    public Void call() throws Exception {
        Message msg = getMsg(msgSize);

        // Single warmup for every topic
        int myPublishCount = 0;
        for (int i = 0; i < numTopics; i++) {
            if (!HedwigBenchmark.amIResponsibleForTopic(startTopicLabel + i, partitionIndex, numPartitions)) {
View Full Code Here
TOP

Related Classes of org.apache.hedwig.protocol.PubSubProtocol.Message

  • org.apache.hedwig.admin.console.HedwigConsole$PubCmd
  • org.apache.hedwig.admin.console.HedwigConsole$PubSubCmd
  • org.apache.hedwig.admin.console.ReadTopic
  • org.apache.hedwig.client.benchmark.BenchmarkPublisher
  • org.apache.hedwig.client.handlers.SubscribeResponseHandler
  • org.apache.hedwig.client.netty.impl.AbstractSubscribeResponseHandler
  • org.apache.hedwig.client.netty.impl.simple.SimpleSubscribeResponseHandler
  • org.apache.hedwig.client.TestPubSubClient
  • org.apache.hedwig.data.MessageFormatter
  • org.apache.hedwig.server.delivery.TestFIFODeliveryManager
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.