Package xbird.storage.io

Examples of xbird.storage.io.Segments


        final double percent = scanPercentage / 100d;
        StopWatch watchdog = new StopWatch("nthreads: " + threads + ", capacity: " + capacity
                + ", scanPercentage: " + percent + ", scanLength: " + scanLength + ", pageSize: "
                + pageSize);

        final Segments pager = makeTempSegments(pageSize);
        final Status[] stats = new Status[threads];
        final ReadWriteLock rwlock = new ReentrantReadWriteLock();

        final boolean isGCLOCK = (algo == ReplacementAlgorithm.GClock);

        _start = _stop = false;
        final Thread[] thrs = new Thread[threads];
        for(int i = 0; i < threads; i++) {
            final Status stat = new Status();
            final int thrdnum = i;
            stats[i] = stat;
            thrs[i] = new Thread() {
                public void run() {
                    while(!_start) {
                        // Spin till Time To Go
                        try {
                            Thread.sleep(1);
                        } catch (Exception e) {
                        }
                    }
                    try {
                        if(lock == null) {
                            if(useRWlock) {
                                if(isGCLOCK) {
                                    runBenchmarkWithZipfDistributionReadWriteLockForGCLOCK(rwlock, thrdnum, stat, pager, cache, capacity, round, percent, scanLength, dist);
                                } else {
                                    runBenchmarkWithZipfDistributionReadWriteLock(rwlock, thrdnum, stat, pager, cache, capacity, round, percent, scanLength, dist);
                                }
                            } else {
                                runBenchmarkWithZipfDistributionSync(thrdnum, stat, pager, cache, capacity, round, percent, scanLength, dist);
                            }
                        } else {
                            if(isGCLOCK) {
                                runBenchmarkWithZipfDistributionLockForGCLOCK(thrdnum, stat, lock, pager, cache, capacity, round, percent, scanLength, dist);
                            } else {
                                runBenchmarkWithZipfDistributionLock(thrdnum, stat, lock, pager, cache, capacity, round, percent, scanLength, dist);
                            }
                        }
                    } catch (Throwable e) {
                        e.printStackTrace();
                    }
                }
            };
        }
        PerfmonService perfmon = new PerfmonService(3000);
        if(enableStat) {
            perfmon.start();
        }
        // Run threads
        for(int i = 0; i < threads; i++) {
            thrs[i].start();
        }
        _start = true;
        try {
            Thread.sleep(TIME_TO_EST);
        } catch (InterruptedException e) {
        }
        _stop = true;
        for(int i = 0; i < threads; i++) {
            thrs[i].join();
        }
        if(enableStat) {
            perfmon.stop();
        }
        long elapsedCpuTime = sunmx.getProcessCpuTime() - startCpuTime;

        long numOps = 0L, cacheMisses = 0L, elapsed = 0L;
        int ioContention = 0;
        for(int i = 0; i < threads; i++) {
            numOps += stats[i].ops;
            cacheMisses += stats[i].miss;
            elapsed += stats[i].mills;
            ioContention += stats[i].iocontention;
        }
        long avgElapsedInSec = (elapsed / threads) / 1000L;
        long opsPerSec = numOps / avgElapsedInSec;

        long hit = numOps - cacheMisses;
        double hitRatio = ((double) hit) / numOps;
        double missRatio = ((double) cacheMisses) / numOps;
        System.err.println("["
                + algo
                + ((algo == ReplacementAlgorithm.NbGClock || algo == ReplacementAlgorithm.NbGClockK) ? ", counter type: "
                        + System.getProperty("xbird.counter", "striped")
                        : "")
                + " - "
                + filename
                + "] synchronization: "
                + (useRWlock ? "rwlock" : (lock == null ? "sync"
                        : ((lock instanceof NoopLock) ? "non-blocking" : "spinlock"))) + ", read: "
                + numOps + ", miss: " + cacheMisses + ", hit ratio: " + hitRatio + ", miss ratio: "
                + missRatio + ", io contention: " + ioContention);
        System.err.println(watchdog.toString());
        System.err.println("number of ops: " + numOps + ", average number of ops per second: "
                + opsPerSec);
        System.err.println("Elapsed cpu time: " + DateTimeFormatter.formatNanoTime(elapsedCpuTime));
        System.err.flush();

        pager.close();
    }
View Full Code Here


        final double percent = scanPercentage / 100d;
        StopWatch watchdog = new StopWatch("nthreads: " + threads + ", capacity: " + capacity
                + ", scanPercentage: " + percent + ", scanLength: " + scanLength + ", pageSize: "
                + pageSize);

        final Segments pager = makeTempSegments(pageSize);
        final Status[] stats = new Status[threads];
        final ReadWriteLock rwlock = new ReentrantReadWriteLock();

        final boolean isGCLOCK = (algo == ReplacementAlgorithm.GClock);

        _start = _stop = false;
        final Thread[] thrs = new Thread[threads];
        for(int i = 0; i < threads; i++) {
            final Status stat = new Status();
            final int thrdnum = i;
            stats[i] = stat;
            thrs[i] = new Thread() {
                public void run() {
                    while(!_start) {
                        // Spin till Time To Go
                        try {
                            Thread.sleep(1);
                        } catch (Exception e) {
                        }
                    }
                    try {
                        if(lock == null) {
                            if(useRWlock) {
                                if(isGCLOCK) {
                                    runBenchmarkWithZipfDistributionReadWriteLockForGCLOCK(rwlock, thrdnum, stat, pager, cache, capacity, round, percent, scanLength, dist);
                                } else {
                                    runBenchmarkWithZipfDistributionReadWriteLock(rwlock, thrdnum, stat, pager, cache, capacity, round, percent, scanLength, dist);
                                }
                            } else {
                                runBenchmarkWithZipfDistributionSync(thrdnum, stat, pager, cache, capacity, round, percent, scanLength, dist);
                            }
                        } else {
                            if(isGCLOCK) {
                                runBenchmarkWithZipfDistributionLockForGCLOCK(thrdnum, stat, lock, pager, cache, capacity, round, percent, scanLength, dist);
                            } else {
                                runBenchmarkWithZipfDistributionLock(thrdnum, stat, lock, pager, cache, capacity, round, percent, scanLength, dist);
                            }
                        }
                    } catch (Throwable e) {
                        e.printStackTrace();
                    }
                }
            };
        }
        PerfmonService perfmon = new PerfmonService(3000);
        if(enableStat) {
            perfmon.start();
        }
        // Run threads
        for(int i = 0; i < threads; i++) {
            thrs[i].start();
        }
        _start = true;
        try {
            Thread.sleep(TIME_TO_EST);
        } catch (InterruptedException e) {
        }
        _stop = true;
        for(int i = 0; i < threads; i++) {
            thrs[i].join();
        }
        if(enableStat) {
            perfmon.stop();
        }
        long elapsedCpuTime = sunmx.getProcessCpuTime() - startCpuTime;

        long numOps = 0L, cacheMisses = 0L, elapsed = 0L;
        int ioContention = 0;
        for(int i = 0; i < threads; i++) {
            numOps += stats[i].ops;
            cacheMisses += stats[i].miss;
            elapsed += stats[i].mills;
            ioContention += stats[i].iocontention;
        }
        long avgElapsedInSec = (elapsed / threads) / 1000L;
        long opsPerSec = numOps / avgElapsedInSec;

        long hit = numOps - cacheMisses;
        double hitRatio = ((double) hit) / numOps;
        double missRatio = ((double) cacheMisses) / numOps;
        System.err.println("["
                + algo
                + ((algo == ReplacementAlgorithm.NbGClock || algo == ReplacementAlgorithm.NbGClockK) ? ", counter type: "
                        + System.getProperty("xbird.counter", "striped")
                        : "")
                + " - "
                + filename
                + "] synchronization: "
                + (useRWlock ? "rwlock" : (lock == null ? "sync"
                        : ((lock instanceof NoopLock) ? "non-blocking" : "spinlock"))) + ", read: "
                + numOps + ", miss: " + cacheMisses + ", hit ratio: " + hitRatio + ", miss ratio: "
                + missRatio + ", io contention: " + ioContention);
        System.err.println(watchdog.toString());
        System.err.println("number of ops: " + numOps + ", average number of ops per second: "
                + opsPerSec);
        System.err.println("Elapsed cpu time: " + DateTimeFormatter.formatNanoTime(elapsedCpuTime));
        System.err.flush();

        pager.close();
    }
View Full Code Here

        final double percent = scanPercentage / 100d;
        StopWatch watchdog = new StopWatch("nthreads: " + threads + ", capacity: " + capacity
                + ", scanPercentage: " + percent + ", scanLength: " + scanLength + ", pageSize: "
                + pageSize);

        final Segments pager = makeTempSegments(pageSize);
        final Status[] stats = new Status[threads];

        final boolean isGCLOCK = (algo == ReplacementAlgorithm.GClock);

        _start = _stop = false;
        final Thread[] thrs = new Thread[threads];
        for(int i = 0; i < threads; i++) {
            final Status stat = new Status();
            final int thrdnum = i;
            stats[i] = stat;
            thrs[i] = new Thread() {
                public void run() {
                    while(!_start) {
                        // Spin till Time To Go
                        try {
                            Thread.sleep(1);
                        } catch (Exception e) {
                        }
                    }
                    try {
                        if(lock == null) {
                            throw new IllegalStateException();
                        } else {
                            if(isGCLOCK) {
                                runBenchmarkWithZipfDistributionLockForGCLOCK(thrdnum, stat, lock, pager, cache, capacity, round, percent, scanLength, dist);
                            } else {
                                switch(algo) {
                                    case BpFullTwoQueue:
                                    case BpLRU:
                                        //runBenchmarkWithZipfDistributionLazySync(thrdnum, stat, lock, pager, cache, capacity, round, percent, scanLength, dist);
                                        runBenchmarkWithZipfDistributionLazySync2(thrdnum, stat, lock, pager, cache, capacity, round, percent, scanLength, dist);
                                        break;
                                    default:
                                        runBenchmarkWithZipfDistributionLock(thrdnum, stat, lock, pager, cache, capacity, round, percent, scanLength, dist);
                                        break;
                                }
                            }
                        }
                    } catch (Throwable e) {
                        e.printStackTrace();
                    }
                }
            };
        }
        PerfmonService perfmon = new PerfmonService(3000);
        if(enableStat) {
            perfmon.start();
        }
        // Run threads
        for(int i = 0; i < threads; i++) {
            thrs[i].start();
        }
        _start = true;
        try {
            Thread.sleep(TIME_TO_EST);
        } catch (InterruptedException e) {
        }
        _stop = true;
        for(int i = 0; i < threads; i++) {
            thrs[i].join();
        }
        if(enableStat) {
            perfmon.stop();
        }
        long elapsedCpuTime = sunmx.getProcessCpuTime() - startCpuTime;

        long numOps = 0L, cacheMisses = 0L, elapsed = 0L;
        int ioContention = 0;
        for(int i = 0; i < threads; i++) {
            numOps += stats[i].ops;
            cacheMisses += stats[i].miss;
            elapsed += stats[i].mills;
            ioContention += stats[i].iocontention;
        }
        long avgElapsedInSec = (elapsed / threads) / 1000L;
        long opsPerSec = numOps / avgElapsedInSec;

        long hit = numOps - cacheMisses;
        double hitRatio = ((double) hit) / numOps;
        double missRatio = ((double) cacheMisses) / numOps;
        System.err.println("["
                + algo
                + ((algo == ReplacementAlgorithm.NbGClock || algo == ReplacementAlgorithm.NbGClockK) ? ", counter type: "
                        + System.getProperty("xbird.counter", "striped")
                        : "")
                + " - "
                + filename
                + "] synchronization: "
                + (useRWlock ? "rwlock" : (lock == null ? "sync"
                        : ((lock instanceof NoopLock) ? "non-blocking"
                                : lock.getClass().getSimpleName()))) + ", read: " + numOps
                + ", miss: " + cacheMisses + ", hit ratio: " + hitRatio + ", miss ratio: "
                + missRatio + ", io contention: " + ioContention);
        System.err.println(watchdog.toString());
        System.err.println("number of ops: " + numOps + ", average number of ops per second: "
                + opsPerSec);
        System.err.println("Elapsed cpu time: " + DateTimeFormatter.formatNanoTime(elapsedCpuTime));
        System.err.flush();

        pager.close();
    }
View Full Code Here

    public void flush(final DbCollection coll, final String docName, final PropertyMap docProps)
            throws IOException {
        final File chunkFile = getChunkFile(coll, docName);
        assert (!chunkFile.exists()) : "file already exists: " + chunkFile.getAbsolutePath();
        final int splen = _strPool.size();
        final Segments paged = new VarSegments(chunkFile, DescriptorType.hash);
        for(int i = 0; i < splen; i++) {// big string
            final byte[] b = _strPool.get(i);
            final int addr = stringKey(i);
            paged.write(addr, b);
        }
        _strPool.clear();
        final long lcclen = _cpointer >> BLOCK_SHIFT;
        assert (lcclen <= Integer.MAX_VALUE) : lcclen;
        final int cclen = Math.min((int) lcclen, _cchunks.length - 1);
        for(int i = 0; i <= cclen; i++) {
            final char[] c = _cchunks[i];
            final byte[] b = compress(compressor, c);
            final long addr = chunkKey(i * DEFAULT_BLOCK_SIZE_L);
            paged.write(addr, b);
            _cchunks[i] = null;
        }
        docProps.setProperty(KEY_STRPOOL_WRITTEN, String.valueOf(splen));
        docProps.setProperty(KEY_CHUNK_WRITTEN, String.valueOf(cclen));
        paged.flush(false);
        close();
        LOG.info("write string chunk file:" + chunkFile.getAbsolutePath());
    }
View Full Code Here

    public void flush(final DbCollection coll, final String docName, final PropertyMap docProps)
            throws IOException {
        final File chunkFile = getChunkFile(coll, docName);
        assert (!chunkFile.exists()) : "file already exists: " + chunkFile.getAbsolutePath();
        final int splen = _strPool.size();
        final Segments paged = new VarSegments(chunkFile, DescriptorType.hash);
        for(int i = 0; i < splen; i++) {// big string
            final String s = _strPool.get(i);
            final byte[] b = compressor.compress(StringUtils.getBytes(s));
            final int addr = stringKey(i);
            paged.write(addr, b);
        }
        _strPool.clear();
        final long lcclen = _cpointer >> BLOCK_SHIFT;
        assert (lcclen <= Integer.MAX_VALUE) : lcclen;
        final int cclen = Math.min((int) lcclen, _cchunks.length - 1);
        for(int i = 0; i <= cclen; i++) {
            final char[] c = _cchunks[i];
            assert (c != null);
            final byte[] b = compress(compressor, c);
            final int addr = chunkKey(i * BLOCK_SIZE);
            paged.write(addr, b);
            _cchunks[i] = null;
        }
        docProps.setProperty(KEY_STRPOOL_WRITTEN, String.valueOf(splen));
        docProps.setProperty(KEY_CHUNK_WRITTEN, String.valueOf(cclen));
        paged.flush(false);
        close();
        LOG.info("write string chunk file:" + chunkFile.getAbsolutePath());
    }
View Full Code Here

    public void flush(final DbCollection coll, final String docName, final PropertyMap docProps)
            throws IOException {
        final File chunkFile = getChunkFile(coll, docName);
        assert (!chunkFile.exists()) : "file already exists: " + chunkFile.getAbsolutePath();
        final int splen = _strPool.size();
        final Segments paged = new VarSegments(chunkFile, DescriptorType.hash);
        for(int i = 0; i < splen; i++) {// big string
            final String s = _strPool.get(i);
            final byte[] b = compressor.compress(StringUtils.getBytes(s));
            final int addr = stringKey(i);
            paged.write(addr, b);
        }
        _strPool.clear();
        final long lcclen = _cpointer >> BLOCK_SHIFT;
        assert (lcclen <= Integer.MAX_VALUE) : lcclen;
        final int cclen = Math.min((int) lcclen, _cchunks.length - 1);
        for(int i = 0; i <= cclen; i++) {
            final char[] c = _cchunks[i];
            assert (c != null);
            final byte[] b = compress(compressor, c);
            final int addr = chunkKey(i * BLOCK_SIZE);
            paged.write(addr, b);
            _cchunks[i] = null;
        }
        docProps.setProperty(KEY_STRPOOL_WRITTEN, String.valueOf(splen));
        docProps.setProperty(KEY_CHUNK_WRITTEN, String.valueOf(cclen));
        paged.flush(false);
        close();
        LOG.info("write string chunk file:" + chunkFile.getAbsolutePath());
    }
View Full Code Here

    public void flush(final DbCollection coll, final String docName, final PropertyMap docProps)
            throws IOException {
        final File chunkFile = getChunkFile(coll, docName);
        assert (!chunkFile.exists()) : "file already exists: " + chunkFile.getAbsolutePath();
        final int splen = _strPool.size();
        final Segments paged = new VarSegments(chunkFile, DescriptorType.hash);
        for(int i = 0; i < splen; i++) {// big string
            final byte[] b = _strPool.get(i);
            final int addr = stringKey(i);
            paged.write(addr, b);
        }
        _strPool.clear();
        final long lcclen = _cpointer >> BLOCK_SHIFT;
        assert (lcclen <= Integer.MAX_VALUE) : lcclen;
        final int cclen = Math.min((int) lcclen, _cchunks.length - 1);
        for(int i = 0; i <= cclen; i++) {
            final char[] c = _cchunks[i];
            final byte[] b = compress(compressor, c);
            final long addr = chunkKey(i * DEFAULT_BLOCK_SIZE_L);
            paged.write(addr, b);
            _cchunks[i] = null;
        }
        docProps.setProperty(KEY_STRPOOL_WRITTEN, String.valueOf(splen));
        docProps.setProperty(KEY_CHUNK_WRITTEN, String.valueOf(cclen));
        paged.flush(false);
        close();
        LOG.info("write string chunk file:" + chunkFile.getAbsolutePath());
    }
View Full Code Here

TOP

Related Classes of xbird.storage.io.Segments

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.