Examples of Frequency


Examples of org.apache.commons.math3.stat.Frequency

        // Generate sample values
        final int sampleSize = 1000;        // Number of deviates to generate
        final int minExpectedCount = 7;     // Minimum size of expected bin count
        long maxObservedValue = 0;
        final double alpha = 0.001;         // Probability of false failure
        Frequency frequency = new Frequency();
        for (int i = 0; i < sampleSize; i++) {
            long value = randomData.nextPoisson(mean);
            if (value > maxObservedValue) {
                maxObservedValue = value;
            }
            frequency.addValue(value);
        }

        /*
         *  Set up bins for chi-square test.
         *  Ensure expected counts are all at least minExpectedCount.
         *  Start with upper and lower tail bins.
         *  Lower bin = [0, lower); Upper bin = [upper, +inf).
         */
        PoissonDistribution poissonDistribution = new PoissonDistribution(mean);
        int lower = 1;
        while (poissonDistribution.cumulativeProbability(lower - 1) * sampleSize < minExpectedCount) {
            lower++;
        }
        int upper = (int) (5 * mean)// Even for mean = 1, not much mass beyond 5
        while ((1 - poissonDistribution.cumulativeProbability(upper - 1)) * sampleSize < minExpectedCount) {
            upper--;
        }

        // Set bin width for interior bins.  For poisson, only need to look at end bins.
        int binWidth = 0;
        boolean widthSufficient = false;
        double lowerBinMass = 0;
        double upperBinMass = 0;
        while (!widthSufficient) {
            binWidth++;
            lowerBinMass = poissonDistribution.cumulativeProbability(lower - 1, lower + binWidth - 1);
            upperBinMass = poissonDistribution.cumulativeProbability(upper - binWidth - 1, upper - 1);
            widthSufficient = FastMath.min(lowerBinMass, upperBinMass) * sampleSize >= minExpectedCount;
        }

        /*
         *  Determine interior bin bounds.  Bins are
         *  [1, lower = binBounds[0]), [lower, binBounds[1]), [binBounds[1], binBounds[2]), ... ,
         *    [binBounds[binCount - 2], upper = binBounds[binCount - 1]), [upper, +inf)
         *
         */
        List<Integer> binBounds = new ArrayList<Integer>();
        binBounds.add(lower);
        int bound = lower + binWidth;
        while (bound < upper - binWidth) {
            binBounds.add(bound);
            bound += binWidth;
        }
        binBounds.add(upper); // The size of bin [binBounds[binCount - 2], upper) satisfies binWidth <= size < 2*binWidth.

        // Compute observed and expected bin counts
        final int binCount = binBounds.size() + 1;
        long[] observed = new long[binCount];
        double[] expected = new double[binCount];

        // Bottom bin
        observed[0] = 0;
        for (int i = 0; i < lower; i++) {
            observed[0] += frequency.getCount(i);
        }
        expected[0] = poissonDistribution.cumulativeProbability(lower - 1) * sampleSize;

        // Top bin
        observed[binCount - 1] = 0;
        for (int i = upper; i <= maxObservedValue; i++) {
            observed[binCount - 1] += frequency.getCount(i);
        }
        expected[binCount - 1] = (1 - poissonDistribution.cumulativeProbability(upper - 1)) * sampleSize;

        // Interior bins
        for (int i = 1; i < binCount - 1; i++) {
            observed[i] = 0;
            for (int j = binBounds.get(i - 1); j < binBounds.get(i); j++) {
                observed[i] += frequency.getCount(j);
            } // Expected count is (mass in [binBounds[i-1], binBounds[i])) * sampleSize
            expected[i] = (poissonDistribution.cumulativeProbability(binBounds.get(i) - 1) -
                poissonDistribution.cumulativeProbability(binBounds.get(i - 1) -1)) * sampleSize;
        }

View Full Code Here

Examples of org.apache.commons.math3.stat.Frequency

            hexString = randomData.nextHexString(0);
            Assert.fail("zero length requested -- expecting MathIllegalArgumentException");
        } catch (MathIllegalArgumentException ex) {
            // ignored
        }
        Frequency f = new Frequency();
        for (int i = 0; i < smallSampleSize; i++) {
            hexString = randomData.nextHexString(100);
            if (hexString.length() != 100) {
                Assert.fail("incorrect length for generated string");
            }
            for (int j = 0; j < hexString.length(); j++) {
                f.addValue(hexString.substring(j, j + 1));
            }
        }
        double[] expected = new double[16];
        long[] observed = new long[16];
        for (int i = 0; i < 16; i++) {
            expected[i] = (double) smallSampleSize * 100 / 16;
            observed[i] = f.getCount(hex[i]);
        }
        TestUtils.assertChiSquareAccept(expected, observed, 0.001);
    }
View Full Code Here

Examples of org.apache.commons.math3.stat.Frequency

            hexString = randomData.nextSecureHexString(0);
            Assert.fail("zero length requested -- expecting MathIllegalArgumentException");
        } catch (MathIllegalArgumentException ex) {
            // ignored
        }
        Frequency f = new Frequency();
        for (int i = 0; i < smallSampleSize; i++) {
            hexString = randomData.nextSecureHexString(100);
            if (hexString.length() != 100) {
                Assert.fail("incorrect length for generated string");
            }
            for (int j = 0; j < hexString.length(); j++) {
                f.addValue(hexString.substring(j, j + 1));
            }
        }
        double[] expected = new double[16];
        long[] observed = new long[16];
        for (int i = 0; i < 16; i++) {
            expected[i] = (double) smallSampleSize * 100 / 16;
            observed[i] = f.getCount(hex[i]);
        }
        TestUtils.assertChiSquareAccept(expected, observed, 0.001);
    }
View Full Code Here

Examples of org.apache.commons.math3.stat.Frequency

        binBounds[0] = min + binSize;
        for (int i = 1; i < binCount - 1; i++) {
            binBounds[i] = binBounds[i - 1] + binSize;  // + instead of * to avoid overflow in extreme case
        }
       
        final Frequency freq = new Frequency();
        for (int i = 0; i < smallSampleSize; i++) {
            final double value = randomData.nextUniform(min, max);
            Assert.assertTrue("nextUniform range", (value > min) && (value < max));
            // Find bin
            int j = 0;
            while (j < binCount - 1 && value > binBounds[j]) {
                j++;
            }
            freq.addValue(j);
        }
      
        final long[] observed = new long[binCount];
        for (int i = 0; i < binCount; i++) {
            observed[i] = freq.getCount(i);
        }
        final double[] expected = new double[binCount];
        for (int i = 0; i < binCount; i++) {
            expected[i] = 1d / binCount;
        }
View Full Code Here

Examples of org.apache.falcon.entity.v0.Frequency

        FeedEntityParser parser = (FeedEntityParser) EntityParserFactory.getParser(EntityType.FEED);
        Feed feed = parser.parse(stream);
        Assert.assertNotNull(feed);

        final LateArrival lateArrival = new LateArrival();
        lateArrival.setCutOff(new Frequency("4", Frequency.TimeUnit.hours));
        feed.setLateArrival(lateArrival);

        StringWriter stringWriter = new StringWriter();
        Marshaller marshaller = EntityType.FEED.getMarshaller();
        marshaller.marshal(feed, stringWriter);
View Full Code Here

Examples of org.apache.falcon.entity.v0.Frequency

            public long getDelay(Frequency delay, int eventNumber) throws FalconException {
                return 0;
            }
        };

        Frequency frequency = new Frequency("minutes(1)");
        Assert.assertEquals(policy.getDurationInMilliSec(frequency), 60000);
        frequency = new Frequency("minutes(15)");
        Assert.assertEquals(policy.getDurationInMilliSec(frequency), 900000);
        frequency = new Frequency("hours(2)");
        Assert.assertEquals(policy.getDurationInMilliSec(frequency), 7200000);
    }
View Full Code Here

Examples of org.apache.falcon.entity.v0.Frequency

    }

    @Test
    public void testExpBackoffPolicy() throws FalconException {
        AbstractRerunPolicy backoff = new ExpBackoffPolicy();
        long delay = backoff.getDelay(new Frequency("minutes(2)"), 2);
        Assert.assertEquals(delay, 480000);

        long currentTime = System.currentTimeMillis();
        delay = backoff.getDelay(new Frequency("minutes(2)"),
                new Date(currentTime - 1 * 4 * 60 * 1000),
                new Date(currentTime + 1 * 60 * 60 * 1000));
        Assert.assertEquals(delay, 1 * 6 * 60 * 1000);

        currentTime = System.currentTimeMillis();
        delay = backoff.getDelay(new Frequency("minutes(1)"),
                new Date(currentTime - 1 * 9 * 60 * 1000),
                new Date(currentTime + 1 * 60 * 60 * 1000));
        Assert.assertEquals(delay, 900000);
    }
View Full Code Here

Examples of org.apache.falcon.entity.v0.Frequency

    }

    @Test
    public void testPeriodicPolicy() throws FalconException, InterruptedException {
        AbstractRerunPolicy periodic = new PeriodicPolicy();
        long delay = periodic.getDelay(new Frequency("minutes(2)"), 2);
        Assert.assertEquals(delay, 120000);
        delay = periodic.getDelay(new Frequency("minutes(2)"), 5);
        Assert.assertEquals(delay, 120000);

        long currentTime = System.currentTimeMillis();
        //Thread.sleep(1000);
        delay = periodic.getDelay(new Frequency("minutes(3)"),
                new Date(currentTime),
                new Date(currentTime + 1 * 60 * 60 * 1000));
        Assert.assertEquals(delay, 180000);
    }
View Full Code Here

Examples of org.apache.falcon.entity.v0.Frequency

        throws FalconException {
        CONTROLS controls = new CONTROLS();
        controls.setConcurrency(String.valueOf(process.getParallel()));
        controls.setExecution(process.getOrder().name());

        Frequency timeout = process.getTimeout();
        long frequencyInMillis = ExpressionHelper.get().evaluate(process.getFrequency().toString(), Long.class);
        long timeoutInMillis;
        if (timeout != null) {
            timeoutInMillis = ExpressionHelper.get().
                    evaluate(process.getTimeout().toString(), Long.class);
View Full Code Here

Examples of org.apache.falcon.entity.v0.Frequency

                        .getResourceAsStream("/config/feed/feed-0.1.xml"));
        feed2.setName("feed2");
        feed2.getLocations().getLocations().get(0).setPath("/data/clicks/${YEAR}/${MONTH}/${DAY}/${HOUR}");
        feed2.getClusters().getClusters().get(0).getLocations().getLocations()
                .get(0).setPath("/data/clicks/${YEAR}/${MONTH}/${DAY}/${HOUR}");
        feed2.setFrequency(new Frequency("hours(1)"));
        try {
            parser.parseAndValidate(feed2.toString());
        } catch (FalconException e) {
            e.printStackTrace();
            Assert.fail("Not expecting exception for same frequency");
        }
        feed2.setFrequency(new Frequency("hours(2)"));
        //expecting exception
        parser.parseAndValidate(feed2.toString());
    }
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.