Examples of StreamingStats

voldemort.store.stats.StreamingStats

Examples of voldemort.store.stats.StreamingStats

            request = builder.build();
        }


        StorageEngine<ByteArray, byte[], byte[]> storageEngine = AdminServiceRequestHandler.getStorageEngine(storeRepository,
                                                                                                             request.getStore());
        StreamingStats streamStats = null;
        if(isJmxEnabled) {
            streamStats = storeRepository.getStreamingStats(storageEngine.getName());
            streamStats.reportNetworkTime(Operation.SLOP_UPDATE, networkTimeNs);
        }
        networkTimeNs = 0;


        ByteArray key = ProtoUtils.decodeBytes(request.getKey());
        VectorClock vectorClock = ProtoUtils.decodeClock(request.getVersion());


        switch(request.getRequestType()) {
            case PUT:
                try {


                    // Retrieve the transform if its exists
                    byte[] transforms = null;
                    if(request.hasTransform()) {
                        transforms = ProtoUtils.decodeBytes(request.getTransform()).get();
                    }


                    // Retrieve the value
                    byte[] value = ProtoUtils.decodeBytes(request.getValue()).get();
                    startNs = System.nanoTime();
                    storageEngine.put(key, Versioned.value(value, vectorClock), transforms);
                    if(isJmxEnabled)
                        streamStats.reportStorageTime(Operation.SLOP_UPDATE,
                                                      Utils.elapsedTimeNs(startNs,
                                                                          System.nanoTime()));
                    if(logger.isTraceEnabled())
                        logger.trace("updateSlopEntries (Streaming put) successful on key:" + key
                                     + " of store: " + request.getStore());
                } catch(ObsoleteVersionException e) {
                    // log and ignore
                    if(logger.isDebugEnabled())
                        logger.debug("updateSlopEntries (Streaming put) threw ObsoleteVersionException, Ignoring.");
                }
                break;
            case DELETE:
                try {
                    startNs = System.nanoTime();
                    storageEngine.delete(key, vectorClock);
                    if(isJmxEnabled)
                        streamStats.reportStorageTime(Operation.SLOP_UPDATE, System.nanoTime()
                                                                             - startNs);


                    if(logger.isTraceEnabled())
                        logger.trace("updateSlopEntries (Streaming delete) successful");
                } catch(ObsoleteVersionException e) {
                    // log and ignore
                    if(logger.isDebugEnabled())
                        logger.debug("updateSlopEntries (Streaming delete) threw ObsoleteVersionException, Ignoring.");
                }
                break;
            default:
                throw new VoldemortException("Unsupported operation ");
        }


        // log progress
        counter++;
        if(isJmxEnabled)
            streamStats.reportStreamingPut(Operation.SLOP_UPDATE);


        if(0 == counter % 100000) {
            long totalTime = (System.currentTimeMillis() - startTime) / 1000;


            if(logger.isDebugEnabled())

View Full Code Here

Examples of voldemort.store.stats.StreamingStats

                                                                                                            .setDescription("Fetch and update")
                                                                                                            .setStatus("Started");
        final StoreDefinition storeDef = metadataStore.getStoreDef(storeName);
        final boolean isReadOnlyStore = storeDef.getType()
                                                .compareTo(ReadOnlyStorageConfiguration.TYPE_NAME) == 0;
        final StreamingStats streamingStats = voldemortConfig.isJmxEnabled() ? storeRepository.getStreamingStats(storeName)
                                                                            : null;


        try {
            asyncService.submitOperation(requestId, new AsyncOperation(requestId,
                                                                       "Fetch and Update") {


                private final AtomicBoolean running = new AtomicBoolean(true);


                @Override
                public void stop() {
                    running.set(false);
                    logger.info("Stopping fetch and update for store " + storeName + " from node "
                                + nodeId + "( " + partitionIds + " )");
                }


                @Override
                public void operate() {
                    AdminClient adminClient = AdminClient.createTempAdminClient(voldemortConfig,
                                                                                metadataStore.getCluster(),
                                                                                voldemortConfig.getClientMaxConnectionsPerNode());
                    try {
                        StorageEngine<ByteArray, byte[], byte[]> storageEngine = getStorageEngine(storeRepository,
                                                                                                  storeName);


                        EventThrottler throttler = new EventThrottler(voldemortConfig.getStreamMaxWriteBytesPerSec());


                        if(isReadOnlyStore) {
                            ReadOnlyStorageEngine readOnlyStorageEngine = ((ReadOnlyStorageEngine) storageEngine);
                            String destinationDir = readOnlyStorageEngine.getCurrentDirPath();
                            logger.info("Fetching files for RO store '" + storeName
                                        + "' from node " + nodeId + " ( " + partitionIds + " )");
                            updateStatus("Fetching files for RO store '" + storeName
                                         + "' from node " + nodeId + " ( " + partitionIds + " )");


                            adminClient.readonlyOps.fetchPartitionFiles(nodeId,
                                                                        storeName,
                                                                        partitionIds,
                                                                        destinationDir,
                                                                        readOnlyStorageEngine.getChunkedFileSet()
                                                                                             .getChunkIdToNumChunks()
                                                                                             .keySet(),
                                                                        running);


                        } else {
                            logger.info("Fetching entries for RW store '" + storeName
                                        + "' from node " + nodeId + " ( " + partitionIds + " )");
                            updateStatus("Fetching entries for RW store '" + storeName
                                         + "' from node " + nodeId + " ( " + partitionIds + " ) ");


                            if(partitionIds.size() > 0) {
                                Iterator<Pair<ByteArray, Versioned<byte[]>>> entriesIterator = adminClient.bulkFetchOps.fetchEntries(nodeId,
                                                                                                                                     storeName,
                                                                                                                                     partitionIds,
                                                                                                                                     filter,
                                                                                                                                     false,
                                                                                                                                     initialCluster,
                                                                                                                                     0);
                                long numTuples = 0;
                                long startTime = System.currentTimeMillis();
                                long startNs = System.nanoTime();
                                while(running.get() && entriesIterator.hasNext()) {


                                    Pair<ByteArray, Versioned<byte[]>> entry = entriesIterator.next();
                                    if(streamingStats != null) {
                                        streamingStats.reportNetworkTime(Operation.UPDATE_ENTRIES,
                                                                         Utils.elapsedTimeNs(startNs,
                                                                                             System.nanoTime()));
                                    }
                                    ByteArray key = entry.getFirst();
                                    Versioned<byte[]> value = entry.getSecond();
                                    startNs = System.nanoTime();
                                    try {
                                        /**
                                         * TODO This also needs to be fixed to
                                         * use the atomic multi version puts
                                         */
                                        storageEngine.put(key, value, null);
                                    } catch(ObsoleteVersionException e) {
                                        // log and ignore
                                        logger.debug("Fetch and update threw Obsolete version exception. Ignoring");
                                    } finally {
                                        if(streamingStats != null) {
                                            streamingStats.reportStreamingPut(Operation.UPDATE_ENTRIES);
                                            streamingStats.reportStorageTime(Operation.UPDATE_ENTRIES,
                                                                             Utils.elapsedTimeNs(startNs,
                                                                                                 System.nanoTime()));
                                        }
                                    }

View Full Code Here

Examples of voldemort.store.stats.StreamingStats

        super();
        this.localStores = new ConcurrentHashMap<String, Store<ByteArray, byte[], byte[]>>();
        this.storageEngines = new ConcurrentHashMap<String, StorageEngine<ByteArray, byte[], byte[]>>();
        if(jmxEnabled) {
            this.streamingStatsMap = new ConcurrentHashMap<String, StreamingStats>();
            this.aggregatedStreamStats = new StreamingStats();
        }
        this.routedStores = new ConcurrentHashMap<String, Store<ByteArray, byte[], byte[]>>();
        this.nodeStores = new ConcurrentHashMap<Pair<String, Integer>, Store<ByteArray, byte[], byte[]>>();
        this.redirectingSocketStores = new ConcurrentHashMap<Pair<String, Integer>, Store<ByteArray, byte[], byte[]>>();
    }

View Full Code Here

Examples of voldemort.store.stats.StreamingStats

                JmxUtils.registerMbean(aggregatedStreamStats,
                                       JmxUtils.createObjectName(this.getClass().getCanonicalName(),
                                                                 "aggregated-streaming-stats"));
            }


            StreamingStats stat = new StreamingStats(aggregatedStreamStats);
            JmxUtils.registerMbean(stat, JmxUtils.createObjectName(this.getClass()
                                                                       .getCanonicalName(),
                                                                   engine.getName()
                                                                           + "-streaming-stats"));
            streamingStatsMap.putIfAbsent(engine.getName(), stat);

View Full Code Here

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.