Examples of DBIterator


Examples of org.iq80.leveldb.DBIterator

      }
      try {
         if (stopped) {
            throw new PersistenceException("LevelDB is stopped");
         }
         DBIterator it = db.iterator(new ReadOptions().fillCache(false));
         try {
            for (it.seekToFirst(); it.hasNext(); ) {
               Map.Entry<byte[], byte[]> entry = it.next();
               entries.add(entry);
               if (entries.size() == batchSize) {
                  final List<Map.Entry<byte[], byte[]>> batch = entries;
                  entries = new ArrayList<Map.Entry<byte[], byte[]>>(batchSize);
                  submitProcessTask(cacheLoaderTask, keyFilter, eacs, taskContext, batch, loadValues, loadMetadata);
               }
            }
            if (!entries.isEmpty()) {
               submitProcessTask(cacheLoaderTask, keyFilter, eacs, taskContext, entries, loadValues, loadMetadata);
            }

            eacs.waitUntilAllCompleted();
            if (eacs.isExceptionThrown()) {
               throw new PersistenceException("Execution exception!", eacs.getFirstException());
            }
         } catch (Exception e) {
            throw new PersistenceException(e);
         } finally {
            try {
               it.close();
            } catch (IOException e) {
               log.warnUnableToCloseDbIterator(e);
            }
         }
      } finally {
View Full Code Here

Examples of org.iq80.leveldb.DBIterator

            }
         }

         List<Long> times = new ArrayList<Long>();
         List<Object> keys = new ArrayList<Object>();
         DBIterator it = expiredDb.iterator(new ReadOptions().fillCache(false));
         long now = ctx.getTimeService().wallClockTime();
         try {
            for (it.seekToFirst(); it.hasNext();) {
               Map.Entry<byte[], byte[]> entry = it.next();

               Long time = (Long) unmarshall(entry.getKey());
               if (time > now)
                  break;
               times.add(time);
               Object key = unmarshall(entry.getValue());
               if (key instanceof List)
                  keys.addAll((List<?>) key);
               else
                  keys.add(key);
            }

            for (Long time : times) {
               expiredDb.delete(marshall(time));
            }

            if (!keys.isEmpty())
               log.debugf("purge (up to) %d entries", keys.size());
            int count = 0;
            for (Object key : keys) {
               byte[] keyBytes = marshall(key);

               byte[] b = db.get(keyBytes);
               if (b == null)
                  continue;
               MarshalledEntry me = (MarshalledEntry) ctx.getMarshaller().objectFromByteBuffer(b);
               // TODO race condition: the entry could be updated between the get and delete!
               if (me.getMetadata() != null && me.getMetadata().isExpired(now)) {
                  // somewhat inefficient to FIND then REMOVE...
                  db.delete(keyBytes);
                  purgeListener.entryPurged(key);
                  count++;
               }

            }
            if (count != 0)
               log.debugf("purged %d entries", count);
         } catch (Exception e) {
            throw new PersistenceException(e);
         } finally {
            try {
               it.close();
            } catch (IOException e) {
               log.warnUnableToCloseDbIterator(e);
            }
         }
      } catch (PersistenceException e) {
View Full Code Here

Examples of org.iq80.leveldb.DBIterator

    }
    byte[] prefix = KeyBuilder.newInstance().add(ENTITY_ENTRY_PREFIX)
        .add(entityType).add(writeReverseOrderedLong(revStartTime))
        .add(entityId).getBytesForLookup();

    DBIterator iterator = null;
    try {
      iterator = db.iterator();
      iterator.seek(prefix);

      return getEntity(entityId, entityType, revStartTime, fields, iterator,
          prefix, prefix.length);
    } finally {
      IOUtils.cleanup(LOG, iterator);
View Full Code Here

Examples of org.iq80.leveldb.DBIterator

          public int compare(byte[] o1, byte[] o2) {
            return WritableComparator.compareBytes(o1, 0, o1.length, o2, 0,
                o2.length);
          }
        });
    DBIterator iterator = null;
    try {
      // look up start times for the specified entities
      // skip entities with no start time
      for (String entityId : entityIds) {
        byte[] startTime = getStartTime(entityId, entityType);
        if (startTime != null) {
          List<EntityIdentifier> entities = startTimeMap.get(startTime);
          if (entities == null) {
            entities = new ArrayList<EntityIdentifier>();
            startTimeMap.put(startTime, entities);
          }
          entities.add(new EntityIdentifier(entityId, entityType));
        }
      }
      for (Entry<byte[], List<EntityIdentifier>> entry :
          startTimeMap.entrySet()) {
        // look up the events matching the given parameters (limit,
        // start time, end time, event types) for entities whose start times
        // were found and add the entities to the return list
        byte[] revStartTime = entry.getKey();
        for (EntityIdentifier entityIdentifier : entry.getValue()) {
          EventsOfOneEntity entity = new EventsOfOneEntity();
          entity.setEntityId(entityIdentifier.getId());
          entity.setEntityType(entityType);
          events.addEvent(entity);
          KeyBuilder kb = KeyBuilder.newInstance().add(ENTITY_ENTRY_PREFIX)
              .add(entityType).add(revStartTime).add(entityIdentifier.getId())
              .add(EVENTS_COLUMN);
          byte[] prefix = kb.getBytesForLookup();
          if (windowEnd == null) {
            windowEnd = Long.MAX_VALUE;
          }
          byte[] revts = writeReverseOrderedLong(windowEnd);
          kb.add(revts);
          byte[] first = kb.getBytesForLookup();
          byte[] last = null;
          if (windowStart != null) {
            last = KeyBuilder.newInstance().add(prefix)
                .add(writeReverseOrderedLong(windowStart)).getBytesForLookup();
          }
          if (limit == null) {
            limit = DEFAULT_LIMIT;
          }
          iterator = db.iterator();
          for (iterator.seek(first); entity.getEvents().size() < limit &&
              iterator.hasNext(); iterator.next()) {
            byte[] key = iterator.peekNext().getKey();
            if (!prefixMatches(prefix, prefix.length, key) || (last != null &&
                WritableComparator.compareBytes(key, 0, key.length, last, 0,
                    last.length) > 0)) {
              break;
            }
            TimelineEvent event = getEntityEvent(eventType, key, prefix.length,
                iterator.peekNext().getValue());
            if (event != null) {
              entity.addEvent(event);
            }
          }
        }
View Full Code Here

Examples of org.iq80.leveldb.DBIterator

   */
  private TimelineEntities getEntityByTime(byte[] base,
      String entityType, Long limit, Long starttime, Long endtime,
      String fromId, Long fromTs, Collection<NameValuePair> secondaryFilters,
      EnumSet<Field> fields) throws IOException {
    DBIterator iterator = null;
    try {
      KeyBuilder kb = KeyBuilder.newInstance().add(base).add(entityType);
      // only db keys matching the prefix (base + entity type) will be parsed
      byte[] prefix = kb.getBytesForLookup();
      if (endtime == null) {
        // if end time is null, place no restriction on end time
        endtime = Long.MAX_VALUE;
      }
      // construct a first key that will be seeked to using end time or fromId
      byte[] first = null;
      if (fromId != null) {
        Long fromIdStartTime = getStartTimeLong(fromId, entityType);
        if (fromIdStartTime == null) {
          // no start time for provided id, so return empty entities
          return new TimelineEntities();
        }
        if (fromIdStartTime <= endtime) {
          // if provided id's start time falls before the end of the window,
          // use it to construct the seek key
          first = kb.add(writeReverseOrderedLong(fromIdStartTime))
              .add(fromId).getBytesForLookup();
        }
      }
      // if seek key wasn't constructed using fromId, construct it using end ts
      if (first == null) {
        first = kb.add(writeReverseOrderedLong(endtime)).getBytesForLookup();
      }
      byte[] last = null;
      if (starttime != null) {
        // if start time is not null, set a last key that will not be
        // iterated past
        last = KeyBuilder.newInstance().add(base).add(entityType)
            .add(writeReverseOrderedLong(starttime)).getBytesForLookup();
      }
      if (limit == null) {
        // if limit is not specified, use the default
        limit = DEFAULT_LIMIT;
      }

      TimelineEntities entities = new TimelineEntities();
      iterator = db.iterator();
      iterator.seek(first);
      // iterate until one of the following conditions is met: limit is
      // reached, there are no more keys, the key prefix no longer matches,
      // or a start time has been specified and reached/exceeded
      while (entities.getEntities().size() < limit && iterator.hasNext()) {
        byte[] key = iterator.peekNext().getKey();
        if (!prefixMatches(prefix, prefix.length, key) || (last != null &&
            WritableComparator.compareBytes(key, 0, key.length, last, 0,
                last.length) > 0)) {
          break;
        }
        // read the start time and entity id from the current key
        KeyParser kp = new KeyParser(key, prefix.length);
        Long startTime = kp.getNextLong();
        String entityId = kp.getNextString();

        if (fromTs != null) {
          long insertTime = readReverseOrderedLong(iterator.peekNext()
              .getValue(), 0);
          if (insertTime > fromTs) {
            byte[] firstKey = key;
            while (iterator.hasNext() && prefixMatches(firstKey,
                kp.getOffset(), key)) {
              iterator.next();
              key = iterator.peekNext().getKey();
            }
            continue;
          }
        }

View Full Code Here

Examples of org.iq80.leveldb.DBIterator

  // warning is suppressed to prevent eclipse from noting unclosed resource
  @SuppressWarnings("resource")
  @VisibleForTesting
  List<String> getEntityTypes() throws IOException {
    DBIterator iterator = null;
    try {
      iterator = getDbIterator(false);
      List<String> entityTypes = new ArrayList<String>();
      iterator.seek(ENTITY_ENTRY_PREFIX);
      while (iterator.hasNext()) {
        byte[] key = iterator.peekNext().getKey();
        if (key[0] != ENTITY_ENTRY_PREFIX[0]) {
          break;
        }
        KeyParser kp = new KeyParser(key,
            ENTITY_ENTRY_PREFIX.length);
        String entityType = kp.getNextString();
        entityTypes.add(entityType);
        byte[] lookupKey = KeyBuilder.newInstance().add(ENTITY_ENTRY_PREFIX)
            .add(entityType).getBytesForLookup();
        if (lookupKey[lookupKey.length - 1] != 0x0) {
          throw new IOException("Found unexpected end byte in lookup key");
        }
        lookupKey[lookupKey.length - 1] = 0x1;
        iterator.seek(lookupKey);
      }
      return entityTypes;
    } finally {
      IOUtils.cleanup(LOG, iterator);
    }
View Full Code Here

Examples of org.iq80.leveldb.DBIterator

    long totalCount = 0;
    long t1 = System.currentTimeMillis();
    try {
      List<String> entityTypes = getEntityTypes();
      for (String entityType : entityTypes) {
        DBIterator iterator = null;
        DBIterator pfIterator = null;
        long typeCount = 0;
        try {
          deleteLock.writeLock().lock();
          iterator = getDbIterator(false);
          pfIterator = getDbIterator(false);
View Full Code Here

Examples of org.iq80.leveldb.DBIterator

      public Object invoke(Object proxy, Method method, Object[] args)
          throws Throwable {
        throw new RuntimeException("forced runtime error");
      }
    };
    DBIterator dbiter = (DBIterator) Proxy.newProxyInstance(
        DBIterator.class.getClassLoader(), new Class[] { DBIterator.class },
        rtExcHandler);
    LeveldbIterator iter = new LeveldbIterator(dbiter);
    for (CallInfo ci : RTEXC_METHODS) {
      Method method = iter.getClass().getMethod(ci.methodName, ci.argTypes);
View Full Code Here

Examples of org.iq80.leveldb.DBIterator

    assertEquals(10002, LeveldbTimelineStore.getStartTimeWriteCacheSize(conf));
  }

  private boolean deleteNextEntity(String entityType, byte[] ts)
      throws IOException, InterruptedException {
    DBIterator iterator = null;
    DBIterator pfIterator = null;
    try {
      iterator = ((LeveldbTimelineStore)store).getDbIterator(false);
      pfIterator = ((LeveldbTimelineStore)store).getDbIterator(false);
      return ((LeveldbTimelineStore)store).deleteNextEntity(entityType, ts,
          iterator, pfIterator, false);
View Full Code Here

Examples of org.iq80.leveldb.DBIterator

        // interval task could potentially be running while we are shutting down so check for that
        if (!isRunAllowed()) {
            return null;
        }

        DBIterator it = levelDBFile.getDb().iterator();

        String keyBuffer = null;
        try {
            String prefix = repositoryName + '\0';
            for (it.seek(keyBuilder(repositoryName, "")); it.hasNext(); it.next()) {
                if (!isRunAllowed()) {
                    break;
                }
                keyBuffer = asString(it.peekNext().getKey());

                if (!keyBuffer.startsWith(prefix)) {
                    break;
                }

                String key = keyBuffer.substring(prefix.length());

                if (key != null) {
                    LOG.trace("getKey [{}]", key);
                    keys.add(key);
                }
            }
        } finally {
            // Make sure you close the iterator to avoid resource leaks.
            it.close();
        }

        return Collections.unmodifiableSet(keys);
    }
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.