Examples of org.nustaq.offheap.FSTAsciiStringOffheapMap

org.nustaq.offheap.FSTAsciiStringOffheapMap
Created by ruedi on 27.06.14. An offheap hashmap. Keys are strings with a given max length. The longer the strings, the slower lookup will be. Values < 50 are ok'ish. The Map can reside on disk (using mmapped files) or just in offheap. Note that changing stored classes will lead to decoding errors in case of persisted files. You need to make use of FST versioning annotation in order to avoid this.

        coder.getConf().registerClass(TestRec.class);
        int klen = 16;
        int MAX = 10000;


        ArrayList<String> keys = new ArrayList<>();
        FSTAsciiStringOffheapMap store = new FSTAsciiStringOffheapMap(klen, FSTAsciiStringOffheapMap.GB, MAX, coder);
        TestRec val = new TestRec();


        //while( true )
        {
            System.out.println("put");
            for (int i = 0; i < MAX; i++) {
                String key = "" + Math.random();
                if (key.length() > klen)
                    key = key.substring(0, klen);
                keys.add(key);
                val.setId(key);
                store.put(key, val);
            }


            store.dumpIndexStats();


            System.out.println("remove");
            for (int i = 0; i < keys.size(); i++) {
                String s = keys.get(i);
                TestRec rec = (TestRec) store.get(s);
                if ( rec != null ) {
                    store.remove(s);
                    Assert.assertTrue(rec.getId().equals(s));
                }
            }
            keys.clear();
            store.dumpIndexStats();
            System.out.println("-----------------------------------");
            System.out.println("-----------------------------------");


//            store.put("p",new TestRec());
//            System.out.println("p-----------------------------------");

View Full Code Here

    public void removeSomeMemMapped() throws Exception {
        FSTCoder coder = createCoder();
        coder.getConf().registerClass(TestRec.class);
        int klen = 16;
        new File("/tmp/test.mmf").delete();
        FSTAsciiStringOffheapMap store = new FSTAsciiStringOffheapMap("/tmp/test.mmf", klen, 2*FSTAsciiStringOffheapMap.MB, MAX, coder);
        fillAll(store);
        System.out.println("size pre "+store.getSize());
        removeAll(MAX, store);
        store.free();
    }

View Full Code Here

        FSTCoder coder = createCoder();
        coder.getConf().registerClass(TestRec.class);
        int klen = 16;
        int MAX = 100000;


        FSTAsciiStringOffheapMap store = new FSTAsciiStringOffheapMap(klen, FSTAsciiStringOffheapMap.GB, MAX, coder);


        for ( int iii = 0; iii < 5; iii++ ) {
            long tim = System.currentTimeMillis();
            TestRec val = new TestRec();
            for ( int i = 0; i < MAX; i++ ) {
                val.setX(i);
                String key = "test:" + i;
                val.setId(key);
                store.put(key, val );
            }
            long dur = System.currentTimeMillis() - tim;
            System.out.println("put need "+ dur +" for "+MAX+" recs. "+(MAX/dur)+" per ms ");
            System.out.println("free: "+store.getFreeMem()/1024/1024);
            Assert.assertTrue(store.getSize() == MAX);


            tim = System.currentTimeMillis();
            for ( int i = 0; i < MAX; i++ ) {


                TestRec rec = (TestRec) store.get("test:"+i);
                if ( rec == null || rec.getX() != i )
                    throw new RuntimeException("error");
            }


            dur = System.currentTimeMillis() - tim;
            System.out.println("get need "+ dur +" for "+MAX+" recs. "+(MAX/dur)+" per ms ");


            long freeMem = store.getFreeMem();
            tim = System.currentTimeMillis();
            for ( int i = 0; i < MAX; i++ ) {
                String key = "test:" + i;
                TestRec rec = (TestRec) store.get(key);
                if ( rec == null || rec.getX() != i )
                    throw new RuntimeException("error");
                rec.someRandomString = "#"+i+"#"+i+"#"+i+"######";
                store.put(key,rec);
            }
            dur = System.currentTimeMillis() - tim;
            System.out.println("update inplace need "+ dur +" for "+MAX+" recs. "+(MAX/dur)+" per ms ");
            Assert.assertTrue(store.getFreeMem() == freeMem); // ensure in place update happened


            store.remove("test:13");
            store.remove("test:999");
            store.remove("unknown");


            freeMem = store.getFreeMem();
            TestRec value = new TestRec();
            value.someRandomString = longString;
            value.someRandomString += value.someRandomString;
            value.setId("test:999");
            store.put("test:999", value);
            TestRec readVal = (TestRec) store.get("test:999");
            Assert.assertTrue(readVal.someRandomString.equals(value.someRandomString));
            Assert.assertTrue(store.getFreeMem() != freeMem); // ensure adding update happened


            mutateRandom(MAX, store);


            tim = System.currentTimeMillis();
            Iterator values = store.values();
            int iterCnt = 0;
            while( values.hasNext() ) {
                Object read = values.next();
                Assert.assertTrue(read != null);
                iterCnt++;
            }
            dur = System.currentTimeMillis() - tim+1;
            System.out.println("iter "+ dur +" for "+iterCnt+" recs. "+(MAX/dur)+" per ms ");
            Assert.assertTrue(iterCnt == store.getSize());


            tim = System.currentTimeMillis();
            Iterator<ByteSource> bvalues = store.binaryValues();
            iterCnt = 0;
            while( bvalues.hasNext() ) {
                ByteSource read = bvalues.next();
                if ( read == null ) {
                    System.out.println("ERROR");
                }
                iterCnt++;
            }
            dur = System.currentTimeMillis() - tim+1;
            System.out.println("bin iter "+ dur +" for "+iterCnt+" recs. "+(MAX/dur)+" per ms ");
            Assert.assertTrue(iterCnt == store.getSize());
        }
        store.free();
    }

View Full Code Here

TOP

Related Classes of org.nustaq.offheap.FSTAsciiStringOffheapMap

offheap.StringOffHeapTest

org.nustaq.offheap.bytez.bytesource.LeftCutStringByteSource

org.nustaq.serialization.simpleapi.DefaultCoder

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.