Examples of voldemort.store.readonly.ReadOnlyStorageEngine

voldemort.store.readonly.ReadOnlyStorageEngine
A read-only store that fronts a big file

            }
            cluster = new ClusterMapper().readCluster(clusterXml);


        }


        final Store<ByteArray, byte[], byte[]> store = new ReadOnlyStorageEngine("test",
                                                                                 searcher,
                                                                                 new RoutingStrategyFactory().updateRoutingStrategy(storeDef,
                                                                                                                                    cluster),
                                                                                 nodeId,
                                                                                 new File(storeDir),
                                                                                 0);


        final AtomicInteger obsoletes = new AtomicInteger(0);
        final AtomicInteger nullResults = new AtomicInteger(0);
        final AtomicInteger totalResults = new AtomicInteger(0);


        final BlockingQueue<String> requestIds = new ArrayBlockingQueue<String>(20000);
        final Executor executor = Executors.newFixedThreadPool(1);


        // if they have given us a file make a request generator that reads from
        // it, otherwise just generate random values
        final int numVals = numValues;
        Runnable requestGenerator;
        if(inputFile == null) {
            requestGenerator = new Runnable() {


                public void run() {
                    System.out.println("Generating random requests.");
                    Random random = new Random();
                    try {
                        while(true)
                            requestIds.put(Integer.toString(random.nextInt(numRequests) % numVals));
                    } catch(InterruptedException e) {
                        e.printStackTrace();
                    }
                }
            };
        } else {
            requestGenerator = new Runnable() {


                public void run() {
                    try {
                        System.out.println("Using request file to generate requests.");
                        BufferedReader reader = new BufferedReader(new FileReader(inputFile),
                                                                   1000000);
                        while(true) {
                            String line = reader.readLine();
                            if(line == null)
                                return;
                            requestIds.put(line.trim());
                        }
                    } catch(Exception e) {
                        e.printStackTrace();
                    }
                }
            };
        }
        executor.execute(requestGenerator);


        final Serializer<Object> keySerializer = new JsonTypeSerializer(JsonTypeDefinition.fromJson("'string'"),
                                                                        true);
        final AtomicInteger current = new AtomicInteger();
        final int progressIncrement = numRequests / 5;
        PerformanceTest readWriteTest = new PerformanceTest() {


            @Override
            public void doOperation(int index) throws Exception {
                try {
                    totalResults.incrementAndGet();
                    int curr = current.getAndIncrement();
                    List<Versioned<byte[]>> results = store.get(new ByteArray(keySerializer.toBytes(requestIds.take())),
                                                                null);
                    if(curr % progressIncrement == 0)
                        System.out.println(curr);


                    if(results.size() == 0)

View Full Code Here

        Assert.assertTrue(versionDir.exists());


        // open store
        @SuppressWarnings("unchecked")
        Serializer<Object> serializer = (Serializer<Object>) new DefaultSerializerFactory().getSerializer(serDef);
        ReadOnlyStorageEngine engine = new ReadOnlyStorageEngine(storeName,
                                                                 searchStrategy,
                                                                 new RoutingStrategyFactory().updateRoutingStrategy(def,
                                                                                                                    cluster),
                                                                 0,
                                                                 storeDir,
                                                                 1);
        Store<Object, Object, Object> store = SerializingStore.wrap(engine,
                                                                    serializer,
                                                                    serializer,
                                                                    serializer);


        // check values
        for(Map.Entry<String, String> entry: values.entrySet()) {
            List<Versioned<Object>> found = store.get(entry.getKey(), null);
            Assert.assertEquals("Incorrect number of results", 1, found.size());
            Assert.assertEquals(entry.getValue(), found.get(0).getValue());
        }


        // also check the iterator - first key iterator...
        try {
            ClosableIterator<ByteArray> keyIterator = engine.keys();
            if(!saveKeys) {
                fail("Should have thrown an exception since this RO format does not support iterators");
            }
            int numElements = 0;
            while(keyIterator.hasNext()) {
                Assert.assertTrue(values.containsKey(serializer.toObject(keyIterator.next().get())));
                numElements++;
            }


            Assert.assertEquals(numElements, values.size());
        } catch(UnsupportedOperationException e) {
            if(saveKeys) {
                fail("Should not have thrown an exception since this RO format does support iterators");
            }
        }


        // ... and entry iterator
        try {
            ClosableIterator<Pair<ByteArray, Versioned<byte[]>>> entryIterator = engine.entries();
            if(!saveKeys) {
                fail("Should have thrown an exception since this RO format does not support iterators");
            }
            int numElements = 0;
            while(entryIterator.hasNext()) {

View Full Code Here

        File versionDir = new File(storeDir, "version-0");
        HdfsFetcher fetcher = new HdfsFetcher();
        fetcher.fetch(nodeFile.getAbsolutePath(), versionDir.getAbsolutePath());


        // Test if we work in the normal collision scenario open store
        ReadOnlyStorageEngine engine = new ReadOnlyStorageEngine(storeName,
                                                                 new CustomBinarySearchStrategy(),
                                                                 new RoutingStrategyFactory().updateRoutingStrategy(def,
                                                                                                                    cluster),
                                                                 0,
                                                                 storeDir,
                                                                 1);
        Store<Object, Object, Object> store = SerializingStore.wrap(engine,
                                                                    serializer,
                                                                    serializer,
                                                                    serializer);


        // check values
        for(Map.Entry<String, String> entry: values.entrySet()) {
            List<Versioned<Object>> found = store.get(entry.getKey(), null);
            Assert.assertEquals("Incorrect number of results", 1, found.size());
            Assert.assertEquals(entry.getValue(), found.get(0).getValue());
        }


        // also check the iterator - first key iterator...
        List<String> valuesLeft2 = Lists.newArrayList(valuesLeft);
        ClosableIterator<ByteArray> keyIterator = engine.keys();
        int numElements = 0;
        while(keyIterator.hasNext()) {
            Object object = serializer.toObject(keyIterator.next().get());
            assertEquals(valuesLeft.remove(object), true);
            Assert.assertTrue(values.containsKey(object));
            numElements++;
        }


        Assert.assertEquals(numElements, values.size());
        Assert.assertEquals(valuesLeft.size(), 0);


        // ... and entry iterator
        ClosableIterator<Pair<ByteArray, Versioned<byte[]>>> entryIterator = engine.entries();
        numElements = 0;
        while(entryIterator.hasNext()) {
            Pair<ByteArray, Versioned<byte[]>> entry = entryIterator.next();
            assertEquals(valuesLeft2.remove(serializer.toObject(entry.getFirst().get())), true);
            Assert.assertEquals(values.get(serializer.toObject(entry.getFirst().get())),

View Full Code Here

                                                                                           "test-readonly-versions"));
        assertEquals(storesToVersions.size(), 2);
        assertEquals(storesToVersions.get("test-readonly-fetchfiles").longValue(), 0);
        assertEquals(storesToVersions.get("test-readonly-versions").longValue(), 0);


        ReadOnlyStorageEngine storeNode0 = (ReadOnlyStorageEngine) getStore(0,
                                                                            "test-readonly-fetchfiles");
        ReadOnlyStorageEngine storeNode1 = (ReadOnlyStorageEngine) getStore(1,
                                                                            "test-readonly-fetchfiles");


        Utils.mkdirs(new File(storeNode0.getStoreDirPath(), "version-10"));
        File newVersionNode1 = new File(storeNode1.getStoreDirPath(), "version-11");
        Utils.mkdirs(newVersionNode1);
        storeNode1.swapFiles(newVersionNode1.getAbsolutePath());


        // Node 0
        // Test current version
        storesToVersions = getAdminClient().readonlyOps.getROCurrentVersion(0,
                                                                            Lists.newArrayList("test-readonly-fetchfiles"));

View Full Code Here

        Map<Integer, Set<Pair<Integer, Integer>>> buckets = ROTestUtils.getNodeIdToAllPartitions(cluster,
                                                                                                 StoreDefinitionUtils.getStoreDefinitionWithName(storeDefs,
                                                                                                                                                 "test-readonly-fetchfiles"),
                                                                                                 true);
        for(Node node: cluster.getNodes()) {
            ReadOnlyStorageEngine store = (ReadOnlyStorageEngine) getStore(node.getId(),
                                                                           "test-readonly-fetchfiles");


            // Create list of buckets ( replica to partition )
            Set<Pair<Integer, Integer>> nodeBucketsSet = buckets.get(node.getId());
            HashMap<Integer, List<Integer>> nodeBuckets = ROTestUtils.flattenPartitionTuples(nodeBucketsSet);


            // Split the buckets into primary and replica buckets
            HashMap<Integer, List<Integer>> primaryNodeBuckets = Maps.newHashMap();
            primaryNodeBuckets.put(0, nodeBuckets.get(0));
            int primaryPartitions = nodeBuckets.get(0).size();


            HashMap<Integer, List<Integer>> replicaNodeBuckets = Maps.newHashMap(nodeBuckets);
            replicaNodeBuckets.remove(0);


            int replicaPartitions = 0;
            for(List<Integer> partitions: replicaNodeBuckets.values()) {
                replicaPartitions += partitions.size();
            }


            // Generate data...
            File newVersionDir = new File(store.getStoreDirPath(), "version-"
                                                                   + Long.toString(versionId));
            Utils.mkdirs(newVersionDir);
            generateROFiles(numChunks, indexSize, dataSize, nodeBuckets, newVersionDir);


            // Swap it...
            store.swapFiles(newVersionDir.getAbsolutePath());


            // Check if everything got mmap-ed correctly...
            HashMap<Object, Integer> chunkIdToNumChunks = store.getChunkedFileSet()
                                                               .getChunkIdToNumChunks();
            for(Object bucket: chunkIdToNumChunks.keySet()) {
                Pair<Integer, Integer> partitionToReplicaBucket = (Pair<Integer, Integer>) bucket;
                Pair<Integer, Integer> replicaToPartitionBucket = Pair.create(partitionToReplicaBucket.getSecond(),
                                                                              partitionToReplicaBucket.getFirst());

View Full Code Here

                int nodeId = entry.getKey();
                Set<Pair<Integer, Integer>> buckets = entry.getValue();


                assertEquals(servers[nodeId].getMetadataStore().getCluster(), cluster);


                ReadOnlyStorageEngine engine = (ReadOnlyStorageEngine) servers[nodeId].getStoreRepository()
                                                                                      .getStorageEngine(storeDef.getName());
                HashMap<Object, Integer> storeBuckets = engine.getChunkedFileSet()
                                                              .getChunkIdToNumChunks();


                for(Pair<Integer, Integer> bucket: buckets) {
                    if(bucket.getFirst() < storeDef.getReplicationFactor())
                        assertEquals(storeBuckets.containsKey(Pair.create(bucket.getSecond(),

View Full Code Here

            throw new VoldemortException("Should be fetching partition files only for read-only stores");
        }


        List<Integer> partitionIds = request.getPartitionIdsList();
        this.partitionIds = partitionIds;
        ReadOnlyStorageEngine storageEngine = AdminServiceRequestHandler.getReadOnlyStorageEngine(metadataStore,
                                                                                                  storeRepository,
                                                                                                  request.getStoreName());
        this.bucketToNumChunks = storageEngine.getChunkedFileSet().getChunkIdToNumChunks();
        this.blockSize = voldemortConfig.getAllProps()
                                        .getLong("partition.buffer.size.bytes",
                                                 voldemortConfig.getAdminSocketBufferSize());
        this.storeDir = new File(storageEngine.getCurrentDirPath());
        this.throttler = new EventThrottler(voldemortConfig.getStreamMaxReadBytesPerSec());
        if(voldemortConfig.isJmxEnabled()) {
            this.streamStats = storeRepository.getStreamingStats(storageEngine.getName());
        } else {
            this.streamStats = null;
        }
        this.partitionIterator = Collections.unmodifiableList(partitionIds).iterator();
        this.fetchStatus = FetchStatus.NEXT_PARTITION;

View Full Code Here

        VAdminProto.GetROCurrentVersionDirResponse.Builder response = VAdminProto.GetROCurrentVersionDirResponse.newBuilder();


        try {
            for(String storeName: storeNames) {


                ReadOnlyStorageEngine store = getReadOnlyStorageEngine(metadataStore,
                                                                       storeRepository,
                                                                       storeName);
                VAdminProto.ROStoreVersionDirMap storeResponse = VAdminProto.ROStoreVersionDirMap.newBuilder()
                                                                                                 .setStoreName(storeName)
                                                                                                 .setStoreDir(store.getCurrentDirPath())
                                                                                                 .build();
                response.addRoStoreVersions(storeResponse);
            }
        } catch(VoldemortException e) {
            response.setError(ProtoUtils.encodeError(errorCodeMapper, e));

View Full Code Here

        VAdminProto.GetROMaxVersionDirResponse.Builder response = VAdminProto.GetROMaxVersionDirResponse.newBuilder();


        try {
            for(String storeName: storeNames) {


                ReadOnlyStorageEngine store = getReadOnlyStorageEngine(metadataStore,
                                                                       storeRepository,
                                                                       storeName);
                File storeDirPath = new File(store.getStoreDirPath());


                if(!storeDirPath.exists())
                    throw new VoldemortException("Unable to locate the directory of the read-only store "
                                                 + storeName);

View Full Code Here

        VAdminProto.GetROStorageFormatResponse.Builder response = VAdminProto.GetROStorageFormatResponse.newBuilder();


        try {
            for(String storeName: storeNames) {


                ReadOnlyStorageEngine store = getReadOnlyStorageEngine(metadataStore,
                                                                       storeRepository,
                                                                       storeName);
                VAdminProto.ROStoreVersionDirMap storeResponse = VAdminProto.ROStoreVersionDirMap.newBuilder()
                                                                                                 .setStoreName(storeName)
                                                                                                 .setStoreDir(store.getReadOnlyStorageFormat()
                                                                                                                   .getCode())
                                                                                                 .build();


                response.addRoStoreVersions(storeResponse);
            }

View Full Code Here

0 1

TOP

Related Classes of voldemort.store.readonly.ReadOnlyStorageEngine

voldemort.client.AdminServiceBasicTest

voldemort.client.rebalance.AdminRebalanceTest

voldemort.performance.ReadOnlyStorePerformanceTest

voldemort.server.http.gui.ReadOnlyStoreManagementServlet

voldemort.server.protocol.admin.AdminServiceRequestHandler

voldemort.server.protocol.admin.FetchPartitionFileStreamRequestHandler

voldemort.server.rebalance.Rebalancer

voldemort.store.readonly.chunk.ChunkedFileSet

voldemort.store.readonly.mr.HadoopStoreBuilderCollisionTest

voldemort.store.readonly.mr.HadoopStoreBuilderTest

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.