Examples of HyperLogLog

com.clearspring.analytics.stream.cardinality.HyperLogLog
inria.fr/flajolet/Publications/FlFuGaMe07.pdf
HLL is an improved version of LogLog that is capable of estimating the cardinality of a set with accuracy = 1.04/sqrt(m) where m = 2^b. So we can control accuracy vs space usage by increasing or decreasing b.
The main benefit of using HLL over LL is that it only requires 64% of the space that LL does to get the same accuracy.
This implementation implements a single counter. If a large (millions) number of counters are required you may want to refer to:
http://dsiutils.dsi.unimi.it/
It has a more complex implmentation of HLL that supports multiple counters in a single object, drastically reducing the java overhead from creating a large number of objects.
This implementation leveraged a javascript implementation that Yammer has been working on:
https://github.com/yammer/probablyjs
io.airlift.stats.cardinality.HyperLogLog

Examples of io.airlift.stats.cardinality.HyperLogLog

            }
        }


        private void add(BlockCursor cursor)
        {
            HyperLogLog instance = HyperLogLog.newInstance(cursor.getSlice());


            if (estimator == null) {
                estimator = instance;
            }
            else {

View Full Code Here

Examples of io.airlift.stats.cardinality.HyperLogLog

        }


        @Override
        public void evaluateFinal(int groupId, BlockBuilder output)
        {
            HyperLogLog estimator = estimators.get(groupId);
            if (estimator == null) {
                output.appendNull();
            }
            else {
                output.appendSlice(estimator.serialize());
            }
        }

View Full Code Here

Examples of io.airlift.stats.cardinality.HyperLogLog

            }
        }


        private void add(long groupId, BlockCursor cursor)
        {
            HyperLogLog instance = HyperLogLog.newInstance(cursor.getSlice());


            HyperLogLog previous = estimators.get(groupId);
            if (previous == null) {
                estimators.set(groupId, instance);
                sizeOfValues += instance.estimatedInMemorySize();
            }
            else {
                sizeOfValues -= previous.estimatedInMemorySize();
                previous.mergeWith(instance);
                sizeOfValues += previous.estimatedInMemorySize();
            }
        }

View Full Code Here

Examples of io.airlift.stats.cardinality.HyperLogLog


    @InputFunction
    @IntermediateInputFunction
    public static void merge(HyperLogLogState state, @SqlType(HyperLogLogType.class) Slice value)
    {
        HyperLogLog input = HyperLogLog.newInstance(value);


        HyperLogLog previous = state.getHyperLogLog();
        if (previous == null) {
            state.setHyperLogLog(input);
            state.addMemoryUsage(input.estimatedInMemorySize());
        }
        else {
            state.addMemoryUsage(-previous.estimatedInMemorySize());
            previous.mergeWith(input);
            state.addMemoryUsage(previous.estimatedInMemorySize());
        }
    }

View Full Code Here

Examples of io.airlift.stats.cardinality.HyperLogLog

    private ApproximateSetAggregation() {}


    @InputFunction
    public static void input(HyperLogLogState state, @SqlType(DoubleType.class) double value)
    {
        HyperLogLog hll = getOrCreateHyperLogLog(state);
        state.addMemoryUsage(-hll.estimatedInMemorySize());
        hll.add(Double.doubleToLongBits(value));
        state.addMemoryUsage(hll.estimatedInMemorySize());
    }

View Full Code Here

Examples of io.airlift.stats.cardinality.HyperLogLog

    }


    @InputFunction
    public static void input(HyperLogLogState state, @SqlType(VarcharType.class) Slice value)
    {
        HyperLogLog hll = getOrCreateHyperLogLog(state);
        state.addMemoryUsage(-hll.estimatedInMemorySize());
        hll.add(value);
        state.addMemoryUsage(hll.estimatedInMemorySize());
    }

View Full Code Here

Examples of io.airlift.stats.cardinality.HyperLogLog

    }


    @InputFunction
    public static void input(HyperLogLogState state, @SqlType(BigintType.class) long value)
    {
        HyperLogLog hll = getOrCreateHyperLogLog(state);
        state.addMemoryUsage(-hll.estimatedInMemorySize());
        hll.add(value);
        state.addMemoryUsage(hll.estimatedInMemorySize());
    }

View Full Code Here

Examples of io.airlift.stats.cardinality.HyperLogLog

        state.addMemoryUsage(hll.estimatedInMemorySize());
    }


    private static HyperLogLog getOrCreateHyperLogLog(HyperLogLogState state)
    {
        HyperLogLog hll = state.getHyperLogLog();
        if (hll == null) {
            hll = HyperLogLog.newInstance(NUMBER_OF_BUCKETS);
            state.setHyperLogLog(hll);
            state.addMemoryUsage(hll.estimatedInMemorySize());
        }
        return hll;
    }

View Full Code Here

Examples of io.airlift.stats.cardinality.HyperLogLog

    }


    @CombineFunction
    public static void combineState(HyperLogLogState state, HyperLogLogState otherState)
    {
        HyperLogLog input = otherState.getHyperLogLog();


        HyperLogLog previous = state.getHyperLogLog();
        if (previous == null) {
            state.setHyperLogLog(input);
            state.addMemoryUsage(input.estimatedInMemorySize());
        }
        else {
            state.addMemoryUsage(-previous.estimatedInMemorySize());
            previous.mergeWith(input);
            state.addMemoryUsage(previous.estimatedInMemorySize());
        }
    }

View Full Code Here

Examples of io.airlift.stats.cardinality.HyperLogLog

    private ApproximateSetAggregation() {}


    @InputFunction
    public static void input(HyperLogLogState state, @SqlType(StandardTypes.DOUBLE) double value)
    {
        HyperLogLog hll = getOrCreateHyperLogLog(state);
        state.addMemoryUsage(-hll.estimatedInMemorySize());
        hll.add(Double.doubleToLongBits(value));
        state.addMemoryUsage(hll.estimatedInMemorySize());
    }

View Full Code Here

0 1 2

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.