Package kafka.message

Examples of kafka.message.MessageAndOffset

182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
        totalFetchTime += lastFetchTime;
        messageIter = messageBuffer.iterator();
        //boolean flag = false;
        Iterator<MessageAndOffset> messageIter2 = messageBuffer
            .iterator();
        MessageAndOffset message = null;
        while (messageIter2.hasNext()) {
          message = messageIter2.next();
          if (message.offset() < currentOffset) {
            //flag = true;
            skipped++;
          } else {
            log.debug("Skipped offsets till : "
                + message.offset());
            break;
          }
        }
        log.debug("Number of offsets to be skipped: " + skipped);
        while(skipped !=0 )
        {
          MessageAndOffset skippedMessage = messageIter.next();
          log.debug("Skipping offset : " + skippedMessage.offset());
          skipped --;
        }

        if (!messageIter.hasNext()) {
          System.out
View Full Code Here
118
119
120
121
122
123
124
125
126
127
128
129
            }
        }

        if (iterator.hasNext())
        {
            MessageAndOffset messageOffset = iterator.next();
            Message message = messageOffset.message();
            key.set(watermark - message.size() - 4);
            value.set(message.payload().array(), message.payload().arrayOffset(), message.payloadSize());
            numProcessedMessages++;
            if (!iterator.hasNext())
            {
View Full Code Here
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
        _reporter.incrCounter(topic, "event-count", _count);
    }
   
    protected boolean get(KafkaETLKey key, BytesWritable value) throws IOException {
        if (_messageIt != null && _messageIt.hasNext()) {
            MessageAndOffset msgAndOffset = _messageIt.next();
           
            ByteBuffer buf = msgAndOffset.message().payload();
            int origSize = buf.remaining();
            byte[] bytes = new byte[origSize];
            buf.get(bytes, buf.position(), origSize);
            value.set(bytes, 0, origSize);
           
            key.set(_index, _offset, msgAndOffset.message().checksum());
           
            _offset += msgAndOffset.offset();  //increase offset
            _count ++;  //increase count
           
            return true;
        }
        else return false;
View Full Code Here
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
        if (emittingMessages.isEmpty()) {
            return EmitState.EMIT_NONE;
        }

        while (true) {
            MessageAndOffset toEmitMsg = emittingMessages.pollFirst();
            if (toEmitMsg == null) {
                return EmitState.EMIT_END;
            }
            Iterable<List<Object>> tups = generateTuples(toEmitMsg.message());
            if (tups != null) {
                for (List<Object> tuple : tups) {
                    collector.emit(tuple, new KafkaMessageId(partition, toEmitMsg.offset()));
                }
                // break;
            } else {
                ack(toEmitMsg.offset());
            }
        }

        // if(!emittingMessages.isEmpty()) {
        // return EmitState.EMIT_MORE;
View Full Code Here
TOP

Related Classes of kafka.message.MessageAndOffset

  • co.gridport.kafka.hadoop.KafkaInputRecordReader
  • com.alibaba.jstorm.kafka.PartitionConsumer
  • com.facebook.presto.kafka.KafkaRecordSet$KafkaRecordCursor
  • com.linkedin.camus.etl.kafka.common.KafkaReader
  • com.pinterest.secor.common.KafkaClient
  • kafka.etl.KafkaETLContext
  • org.apache.twill.internal.kafka.client.SimpleKafkaConsumer$ConsumerThread
  • storm.kafka.bolt.KafkaBoltTest
  • storm.kafka.TestUtils
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.