Examples of DbusEventIterator


Examples of com.linkedin.databus.core.DbusEventBuffer.DbusEventIterator

        LOG.info("Checkpoint = " + checkPoint);


        if (_eventSeen)
        {
          DbusEventIterator iter = _eventBuffer.acquireIterator("EventLogReader:firstEvent");

          try
          {
            DbusEvent event = iter.next();
            String firstEventContent = event.toString();
            // if we didn't wrap the buffer, and the first event is not an eop marker, delete the first window
            if (_firstEventContent.equalsIgnoreCase(firstEventContent) && !event.isEndOfPeriodMarker())
            {
              long result = _eventBuffer.deleteFirstWindow();
              if (result < 0)
              {
                throw new RuntimeException("eventBuffer could not delete first window");
              }
            }
          }
          finally
          {
            _eventBuffer.releaseIterator(iter);
          }

          if (_lastEopOffset >=0)
          {
            iter = _eventBuffer.new DbusEventIterator(this._lastEopOffset, _eventBuffer.getTail(),
                                                      "EventLogReader:lastEOP");
            try
            {
              DbusEvent event = iter.next();
              if (!event.isEndOfPeriodMarker())
              {
                throw new RuntimeException("Tried reading end of period marker, but failed");

              }

              if (iter.hasNext())
              {
                _eventBuffer.setTail(iter.getCurrentPosition());
              }


            }
            catch (NoSuchElementException e)
View Full Code Here

Examples of com.linkedin.databus.core.DbusEventBuffer.DbusEventIterator

    LOG.info("Head:" + dbusBuf.getHead() + ",Tail:" +dbusBuf.getTail());
    assertEquals("Head Check",0,dbusBuf.getHead());
    assertEquals("Tail Check",903,dbusBuf.getTail());

    // Remove the first event
    DbusEventIterator itr = dbusBuf.acquireIterator("dummy");
    assertTrue(itr.hasNext());
    DbusEvent event = itr.next();
    assertTrue(event.isValid());
    itr.remove();
    LOG.info("Head:" + parser.toString(dbusBuf.getHead()) + ",Tail:" + parser.toString(dbusBuf.getTail()));
    assertEquals("Head Check",61,dbusBuf.getHead());
    assertEquals("Tail Check",903,dbusBuf.getTail());

    for (DbusEvent e : events)
    {
      assertTrue("invalid event", e.isValid());
    }

    // set up the ReadChannel with 2 events
    ByteArrayOutputStream oStream = new ByteArrayOutputStream();
    WritableByteChannel oChannel = Channels.newChannel(oStream);
    for (int i = 0; i < 2; ++i)
    {
      ((DbusEventInternalReadable)events.get(i)).writeTo(oChannel,Encoding.BINARY);
    }
    byte[] writeBytes = oStream.toByteArray();
    ByteArrayInputStream iStream = new ByteArrayInputStream(writeBytes);
    final ReadableByteChannel rChannel = Channels.newChannel(iStream);

    // Create a Thread to call readEvents on the channel
    Runnable writer = new Runnable() {
      @Override
      public void run()
      {
        try
        {
          dbusBuf.readEvents(rChannel);
        } catch (InvalidEventException ie) {
          ie.printStackTrace();
          throw new RuntimeException(ie);
        }
      }
    };

    Thread writerThread = new Thread(writer);
    writerThread.start();

    //Check if the thread is alive (blocked) and head/tail is not overlapped
    trySleep(1000);
    assertTrue(writerThread.isAlive());
    LOG.info("Head:" + parser.toString(dbusBuf.getHead()) + ",Tail:" + parser.toString(dbusBuf.getTail()));
    assertEquals("Head Check",61,dbusBuf.getHead());
    assertEquals("Tail Check",2048,dbusBuf.getTail()); //GenId set here but tail is not yet overlapped

    //Read the next event to unblock the writer
    event = itr.next();
    assertTrue(event.isValid());
    itr.remove();
    try
    {
      writerThread.join(1000);
    } catch (InterruptedException ie) {
      ie.printStackTrace();
    }
    LOG.info("Head:" + parser.toString(dbusBuf.getHead()) + ",Tail:" + parser.toString(dbusBuf.getTail()));

    assertFalse(writerThread.isAlive());
    assertEquals("Head Check",132,dbusBuf.getHead());
    assertEquals("Tail Check",2119,dbusBuf.getTail());

    while (itr.hasNext())
    {
      assertTrue(itr.next().isValid(true));
      itr.remove();
    }
    LOG.info("Head:" + parser.toString(dbusBuf.getHead()) + ",Tail:" + parser.toString(dbusBuf.getTail()));
    assertEquals("Head Check",dbusBuf.getHead(),dbusBuf.getTail());
  }
View Full Code Here

Examples of com.linkedin.databus.core.DbusEventBuffer.DbusEventIterator

    dbuf.appendEvent(key, dbusEventInfo, null);
    dbuf.endEvents(1235);

    //Iterate and verify if the both the upsert and delete events exist
    DbusEventIterator eventIterator = dbuf.acquireIterator("eventIterator");

    //Read EOP
    assert (eventIterator.hasNext());
    eventIterator.next();

    //Read upsert event
    assert (eventIterator.hasNext());
    DbusEventInternalWritable e = eventIterator.next();
    assertEquals(DbusOpcode.UPSERT,e.getOpcode());
    if(type.equals("STRING"))
      assertEquals(stringKey.getBytes(Charset.defaultCharset()),e.keyBytes());
    else if(type.equals("LONG"))
      assertEquals(longKey,e.key());
    assertEquals(1234,e.sequence());

    //Read EOP
    assert (eventIterator.hasNext());
    eventIterator.next();

    //Read delete event
    assert(eventIterator.hasNext());
    e = eventIterator.next();
    assertEquals(DbusOpcode.DELETE, e.getOpcode());
    if(type.equals("STRING"))
      assertEquals(stringKey.getBytes(Charset.defaultCharset()),e.keyBytes());
    else if(type.equals("LONG"))
      assertEquals(longKey,e.key());
    assertEquals(1235,e.sequence());


    //Read EOP
    assert (eventIterator.hasNext());
    eventIterator.next();
  }
View Full Code Here

Examples of com.linkedin.databus.core.DbusEventBuffer.DbusEventIterator

      dbuf.endEvents(i);
    }

    long minDbusEventBufferScn = dbuf.getMinScn();
    long expectedScn = minDbusEventBufferScn;
    DbusEventIterator eventIterator = dbuf.acquireIterator("eventIterator");
    DbusEventInternalWritable e= null;
    int state = 0; // searching for min scn
    long entryNum = 1;
    while (eventIterator.hasNext())
    {
      e = eventIterator.next();
      if (state == 0 && (e.sequence() >= minDbusEventBufferScn))
      {
        state = 1; // found min scn
      }

      if (state == 1)
      {
        assertEquals(expectedScn, e.sequence());
        long ts = e.timestampInNanos();
        assertEquals(ts, timeStamp + entryNum*1000*1000);
        byte[] eventBytes = new byte[e.valueLength()];
        e.value().get(eventBytes);
        assertEquals(eventBytes, testDataMap.get(entryNum).value.getBytes(Charset.defaultCharset()));
        entryNum++;
        e = eventIterator.next();
        assertTrue(e.isEndOfPeriodMarker());
        assertEquals(expectedScn, e.sequence());
        expectedScn++;
      }
    }
    dbuf.releaseIterator(eventIterator);
    Assert.assertEquals(entryNum, numEntries);

    // Reset the buffer with prevScn = 3, and verify
    final long prevScn = 3;
    dbuf.reset(prevScn);
    assertTrue(dbuf.empty());
    assertEquals(-1, dbuf.getMinScn());
    assertEquals(prevScn, dbuf.getPrevScn());
    testDataMap.clear();
    long currentScn = prevScn;

    // Add more events and make sure that the new events are in the buffer.
    //dbuf.start(0);
    valueLength = 25;
    numEntries = 5;
    // Now add entries to and make sure that they appear.
    for (long i=1; i < numEntries; ++i)
    {
      // LOG.info("Iteration:"+i);
      DbusEventKey key = new DbusEventKey(RngUtils.randomLong());
      String value = RngUtils.randomString(valueLength);
      dbuf.startEvents();
      long ts = timeStamp + (i*1200*1000);
      assertTrue(dbuf.appendEvent(key, pPartitionId, lPartitionId,ts, srcId, schemaId, value.getBytes(Charset.defaultCharset()), false));
      testDataMap.put(i, new KeyValue(key, value));
      dbuf.endEvents(currentScn + i);
    }

    minDbusEventBufferScn = dbuf.getMinScn();
    assertEquals(minDbusEventBufferScn, prevScn+1);
    expectedScn = minDbusEventBufferScn;
    eventIterator = dbuf.acquireIterator("eventIterator2");
    state = 0; // searching for min scn
    entryNum = 1;
    while (eventIterator.hasNext())
    {
      e = eventIterator.next();
      if (state == 0)
      {
        if (e.sequence() >= minDbusEventBufferScn)
        {
          state = 1; // found min scn
        }
      }

      if (state == 1)
      {
        assertEquals(expectedScn, e.sequence());
        long ts = e.timestampInNanos();
        assertEquals(ts, timeStamp + entryNum*1200*1000);
        byte[] eventBytes = new byte[e.valueLength()];
        e.value().get(eventBytes);
        assertEquals(eventBytes, testDataMap.get(entryNum).value.getBytes(Charset.defaultCharset()));
        entryNum++;
        e = eventIterator.next();
        assertTrue(e.isEndOfPeriodMarker());
        assertEquals(expectedScn, e.sequence());
        expectedScn++;
      }
    }
View Full Code Here

Examples of com.linkedin.databus.core.DbusEventBuffer.DbusEventIterator

      dbuf.endEvents(i);
    }

    long minDbusEventBufferScn = dbuf.getMinScn();
    long expectedScn = minDbusEventBufferScn;
    DbusEventIterator eventIterator = dbuf.acquireIterator("eventIterator");
    DbusEventInternalWritable e= null;
    int state = 0; // searching for min scn
    while (eventIterator.hasNext())
    {
      e = eventIterator.next();
      if (state == 0 && (e.sequence() >= minDbusEventBufferScn))
      {
        state = 1; // found min scn
      }

      if (state == 1)
      {
        assertEquals(expectedScn, e.sequence());
        long ts = e.timestampInNanos();
        e = eventIterator.next();
        assertTrue(e.isEndOfPeriodMarker());
        assertEquals(expectedScn, e.sequence());
        assertEquals(ts, e.timestampInNanos());
        expectedScn = (expectedScn + 1)%numEntries;
      }
View Full Code Here

Examples of com.linkedin.databus.core.DbusEventBuffer.DbusEventIterator

      dbuf.endEvents(i);
    }

    final long minDbusEventBufferScn = dbuf.getMinScn();
    long expectedScn = minDbusEventBufferScn;
    DbusEventIterator eventIterator = dbuf.acquireIterator("eventIterator");
    DbusEventInternalWritable e= null;
    int state = 0; // searching for min scn
    DbusEventIterator copyIterator = eventIterator.copy(null, "copyIterator");
    while (eventIterator.hasNext())
    {
      e = eventIterator.next();
      if (state == 0 && (e.sequence() >= minDbusEventBufferScn))
      {
        state = 1; // found min scn
      }

      if (state == 1)
      {
        assertEquals(expectedScn, e.sequence());
        e = eventIterator.next();
        assertTrue(e.isEndOfPeriodMarker());
        assertEquals(expectedScn, e.sequence());
        expectedScn = (expectedScn + 1)%numEntries;
      }
    }

    dbuf.releaseIterator(eventIterator);

    state = 0;
    expectedScn = minDbusEventBufferScn;
    while (copyIterator.hasNext())
    {
      e = copyIterator.next();
      if (state == 0 && (e.sequence() >= minDbusEventBufferScn))
      {
        state = 1; // found min scn
      }

      if (state == 1)
      {
        assertEquals(expectedScn, e.sequence());
        e = copyIterator.next();
        assertTrue(e.isEndOfPeriodMarker());
        assertEquals(expectedScn, e.sequence());
        expectedScn = (expectedScn + 1)%numEntries;
      }
    }
View Full Code Here

Examples of com.linkedin.databus.core.DbusEventBuffer.DbusEventIterator

    int messageSize = 0;
    long lastWindowScn = 0;
    checkDbusEventBuffer.clear();
    checkDbusEventBuffer.readEvents(readChannel);
    LOG.debug("Reading events");
    DbusEventIterator eventIterator = checkDbusEventBuffer.acquireIterator("check");
    DbusEventInternalWritable e=null;
    while (eventIterator.hasNext())
    {
      e = eventIterator.next();
      //LOG.info(e.scn()+"," + e.windowScn());
      messageSize += e.size();
      lastWindowScn = e.sequence();
    }
    assertTrue(messageSize <= batchFetchSize);
    LOG.debug("Reading events 2");
    // now we regress
    cp.setWindowScn(lastWindowScn-5);
    cp.setWindowOffset(0);
    checkDbusEventBuffer.releaseIterator(eventIterator);
    LOG.debug("Reading events 3");
    writeFile.delete();
    writeFile = File.createTempFile("test", ".dbus", directory);

    try
    {
      writeChannel = Utils.openChannel(writeFile, true);
      StreamEventsArgs args = new StreamEventsArgs(batchFetchSize);
      dbuf.streamEvents(cp, writeChannel, args);
    }
    catch (ScnNotFoundException e1)
    {
      LOG.error("mainDbus threw ScnNotFound exception");
    }
    LOG.debug("mainDbus Read status a = " + dbuf.getReadStatus());
    assertEquals(0, dbuf.getReadStatus());

    LOG.debug("Reading events 4");
    LOG.debug(writeFile.canRead());

    readChannel = Utils.openChannel(writeFile, false);
    checkDbusEventBuffer.clear();
    messageSize = 0;
    lastWindowScn = 0;
    checkDbusEventBuffer.readEvents(readChannel);
    LOG.debug("Reading events 5");

    eventIterator = checkDbusEventBuffer.acquireIterator("eventIterator");
    LOG.debug("Reading events 6");

    while (eventIterator.hasNext())
    {
      e = eventIterator.next();
      //LOG.info(e.scn()+"," + e.windowScn());
      //assertEquals(startScn+messageOffset + messageNum, e.scn());
      messageSize += e.size();
      lastWindowScn = e.sequence();
      //LOG.info("Reading events...");
View Full Code Here

Examples of com.linkedin.databus.core.DbusEventBuffer.DbusEventIterator

    DbusEventAppender appender = new DbusEventAppender(events, dbusBuf, null, 1.0, false, -1);
    appender.run();

    log.info("verify new iterator");
    DbusEventIterator iter1 =
        dbusBuf.acquireIterator("testInternalIteratorSingleBufFull");

    log.debug("it1=" + iter1);
    Assert.assertEquals(iter1.getCurrentPosition(), dbusBuf.getHead());
    Assert.assertEquals(iter1._iteratorTail.getPosition(), dbusBuf.getTail());
    Assert.assertEquals(dbusBuf._busyIteratorPool.size(), 1);
    Assert.assertTrue(iter1.hasNext());
    DbusEvent e = iter1.next();
    Assert.assertTrue(e.isEndOfPeriodMarker());
    Assert.assertTrue(iter1.hasNext());
    dbusBuf.assertBuffersLimits();

    log.info("make sure we can read some events");
    readAndCompareIteratorEvents(iter1, events, 0, 6, true, true, true);
    log.debug("after read: " + dbusBuf.toShortString());
    log.debug(iter1);

    log.info("append more windows");
    final Vector<DbusEvent> events2 = new Vector<DbusEvent>();
    generator = new DbusEventGenerator(200);
    generator.generateEvents(2, 1, 120, 39, events2);
    appender = new DbusEventAppender(events2, dbusBuf, null, 1.0, false, -1);
    appender.run();
    log.debug("after 2 more events added: " + dbusBuf.toShortString());
    log.debug(iter1);

    readAndCompareIteratorEvents(iter1, events2, 0, 2, true, false, true);
    log.debug("after 2 more events read: " + dbusBuf.toShortString());
    log.debug(iter1);
    dbusBuf.assertBuffersLimits();

    // create another iterator - make sure it can read too
    DbusEventIterator iter2 = dbusBuf.acquireIterator("testInternalIteratorSingleBufFull2");

    long iCWP = iter2.getCurrentPosition();
    long head = dbusBuf.getBufferPositionParser().sanitize(dbusBuf.getHead(), dbusBuf.getBuffer());
    Assert.assertEquals(iCWP, head);
    Assert.assertEquals(iter2._iteratorTail.getPosition(), dbusBuf.getTail());
    Assert.assertEquals(dbusBuf._busyIteratorPool.size(), 2);
    Assert.assertTrue(iter2.hasNext());

    log.debug("iter2=" + iter2);
    readAndCompareIteratorEvents(iter2, events2, 0, 2, true, false, true); // read same events and don't remove
    dbusBuf.releaseIterator(iter2);
    dbusBuf.assertBuffersLimits();
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.