Package kafka.javaapi.message

Examples of kafka.javaapi.message.ByteBufferMessageSet

62
63
64
65
66
67
68
69
70
71
72
            String instanceId = (String) meta.get("instanceId");
            if(!_config.forceFromStart || instanceId.equals(_topologyInstanceId)) {
                SimpleConsumer consumer = _connections.getConsumer(partition);
                long offset = (Long) meta.get("offset");
                long nextOffset = (Long) meta.get("nextOffset");
                ByteBufferMessageSet msgs = consumer.fetch(new FetchRequest(_config.topic, partition % partitionsPerHost, offset, _config.fetchSizeBytes));
                for(MessageAndOffset msg: msgs) {
                    if(offset == nextOffset) break;
                    if(offset > nextOffset) {
                        throw new RuntimeException("Error when re-emitting batch. overshot the end offset");
                    }
View Full Code Here
29
30
31
32
33
34
35
36
37
38
39
         } else {
             long startTime = -1;
             if(config.forceFromStart) startTime = config.startOffsetTime;
             offset = consumer.getOffsetsBefore(config.topic, partition % hosts.partitionsPerHost, startTime, 1)[0];
         }
         ByteBufferMessageSet msgs;
         try {
            msgs = consumer.fetch(new FetchRequest(config.topic, partition % hosts.partitionsPerHost, offset, config.fetchSizeBytes));
         } catch(Exception e) {
             if(e instanceof ConnectException) {
                 throw new FailedFetchException(e);
View Full Code Here
119
120
121
122
123
124
125
126
127
128
129
130
131
        boolean gotNext = get(key, value);

        if(_response != null) {
            Iterator<ByteBufferMessageSet> iter = _response.iterator();
            while ( !gotNext && iter.hasNext()) {
                ByteBufferMessageSet msgSet = iter.next();
                if ( hasError(msgSet)) return false;
                _messageIt =  (Iterator<MessageAndOffset>) msgSet.iterator();
                gotNext = get(key, value);
            }
        }
        return gotNext;
    }
View Full Code Here
62
63
64
65
66
67
68
69
70
71
72
                                                       KafkaProperties.connectionTimeOut,
                                                       KafkaProperties.kafkaProducerBufferSize);

    System.out.println("Testing single fetch");
    FetchRequest req = new FetchRequest(KafkaProperties.topic2, 0, 0L, 100);
    ByteBufferMessageSet messageSet = simpleConsumer.fetch(req);
    printMessages(messageSet);

    System.out.println("Testing single multi-fetch");
    req = new FetchRequest(KafkaProperties.topic2, 0, 0L, 100);
    List<FetchRequest> list = new ArrayList<FetchRequest>();
View Full Code Here
96
97
98
99
100
101
102
103
104
105
106
      Message message = new Message(bytes);
      list.add(message);
    }
    // send events
    System.out.println(" send " + list.size() + " " + _topic + " count events to " + _uri);
    _producer.send(_topic, new ByteBufferMessageSet(kafka.message.NoCompressionCodec$.MODULE$, list));

    // close the producer
    _producer.close();
   
    // generate offset files
View Full Code Here
69
70
71
72
73
74
75
76
77
78
79
      for(int i = 0; i < batchSize; i++)
      {
        Message message = new Message(new byte[messageSize]);
        messageList.add(message);
      }
      ByteBufferMessageSet set = new ByteBufferMessageSet(kafka.message.CompressionCodec$.MODULE$.getCompressionCodec(compression), messageList);
      producer.send(topic, random.nextInt(numParts), set);
      bytesSent.getAndAdd(batchSize * messageSize);
      messagesSent.getAndAdd(messageList.size());
    }
  }
View Full Code Here
46
47
48
49
50
51
52
53
54
55
56
  }

  protected void sendMsgList()
  {
    if (msgList.size() > 0) {
      ByteBufferMessageSet msgSet = new ByteBufferMessageSet(kafka.message.NoCompressionCodec$.MODULE$, msgList);
      producer.send(topic, msgSet);
      msgList.clear();
      totalSize = 0;
    }
  }
View Full Code Here
80
81
82
83
84
85
86
87
88
//            long startOffset = getOffset(topic, partition, config.startOffsetTime);
//            offset = startOffset;
//          }
          throw new KafkaException("fetch data from kafka topic["+topic+"] partition["+partition+"] error:" + code);
        }else {
          ByteBufferMessageSet msgs =  fetchResponse.messageSet(topic, partition);
          return msgs;
        }
  }
View Full Code Here
113
114
115
116
117
118
119
120
121
122
123
        // }
    }

    private void fillMessages() {

        ByteBufferMessageSet msgs;
        try {
            msgs = consumer.fetchMessages(partition.getPartition(), emittingOffset + 1);
            LOG.debug("fetch message from offset " + emittingOffset);
            for (MessageAndOffset msg : msgs) {
                emittingMessages.add(msg);
View Full Code Here
TOP

Related Classes of kafka.javaapi.message.ByteBufferMessageSet

  • co.cask.cdap.logging.kafka.KafkaConsumer
  • com.alibaba.jstorm.kafka.MessageConsumer
  • com.alibaba.jstorm.kafka.PartitionConsumer
  • com.linkedin.camus.etl.kafka.common.KafkaReader
  • kafka.bridge.hadoop.KafkaRecordWriter
  • kafka.etl.impl.DataGenerator
  • kafka.etl.KafkaETLContext
  • kafka.examples.SimpleConsumerDemo
  • kafka.perf.producer.Producer
  • org.apache.twill.internal.kafka.client.SimpleKafkaConsumer$ConsumerThread
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.