Package org.apache.falcon.entity.v0

Examples of org.apache.falcon.entity.v0.Frequency


        throws FalconException {
        CONTROLS controls = new CONTROLS();
        controls.setConcurrency(String.valueOf(process.getParallel()));
        controls.setExecution(process.getOrder().name());

        Frequency timeout = process.getTimeout();
        long frequencyInMillis = ExpressionHelper.get().evaluate(process.getFrequency().toString(), Long.class);
        long timeoutInMillis;
        if (timeout != null) {
            timeoutInMillis = ExpressionHelper.get().
                evaluate(process.getTimeout().toString(), Long.class);
View Full Code Here


                        + entity.getName() + "), ignoring failed retries");
                return;
            }

            int attempts = retry.getAttempts();
            Frequency delay = retry.getDelay();
            PolicyType policy = retry.getPolicy();
            int intRunId = Integer.parseInt(runId);

            if (attempts > intRunId) {
                AbstractRerunPolicy rerunPolicy = RerunPolicyFactory.getRetryPolicy(policy);
View Full Code Here

    @Override
    public void run() {
        int attempt = 1;
        AbstractRerunPolicy policy = new ExpBackoffPolicy();
        Frequency frequency = new Frequency("minutes(1)");
        while (true) {
            try {
                T message;
                try {
                    message = handler.takeFromQueue();
View Full Code Here

                        .getResourceAsStream("/config/feed/feed-0.1.xml"));
        feed2.setName("feed2");
        feed2.getLocations().getLocations().get(0).setPath("/data/clicks/${YEAR}/${MONTH}/${DAY}/${HOUR}");
        feed2.getClusters().getClusters().get(0).getLocations().getLocations()
                .get(0).setPath("/data/clicks/${YEAR}/${MONTH}/${DAY}/${HOUR}");
        feed2.setFrequency(new Frequency("hours(1)"));
        try {
            parser.parseAndValidate(feed2.toString());
        } catch (FalconException e) {
            e.printStackTrace();
            Assert.fail("Not expecting exception for same frequency");
        }
        feed2.setFrequency(new Frequency("hours(2)"));
        //expecting exception
        parser.parseAndValidate(feed2.toString());
    }
View Full Code Here

        storeEntity(EntityType.FEED, "imp-click-join2");
        storeEntity(EntityType.PROCESS, "sample");
        Process process = ConfigurationStore.get().get(EntityType.PROCESS, "sample");
        Process otherProcess = (Process) process.copy();
        otherProcess.setName("sample2");
        otherProcess.setFrequency(new Frequency("days(1)"));
        ConfigurationStore.get().remove(EntityType.PROCESS,
                otherProcess.getName());
        ConfigurationStore.get().publish(EntityType.PROCESS, otherProcess);

        fs.mkdirs(instanceLogPath);
View Full Code Here

                if (entity == null) {
                    LOG.warn("No entity for " + key);
                    continue;
                }
                Date startTime = EntityUtil.getStartTime(entity, cluster);
                Frequency frequency = EntityUtil.getFrequency(entity);
                TimeZone timeZone = EntityUtil.getTimeZone(entity);
                Date nextStart = EntityUtil.getNextStartTime(startTime, frequency, timeZone, now);
                if (nextStart.after(windowEndTime)) {
                    continue;
                }
                ConcurrentMap<Date, Date> pendingInstances = pendingJobs.get(key);
                while (!nextStart.after(windowEndTime)) {
                    if (pendingInstances == null) {
                        pendingJobs.putIfAbsent(key, new ConcurrentHashMap<Date, Date>());
                        pendingInstances = pendingJobs.get(key);
                    }
                    Long latency = monitoredEntities.get(key);
                    if (latency == null) {
                        break;
                    }
                    // 1.5 times latency is when it is supposed to have breached
                    pendingInstances.putIfAbsent(nextStart, new Date(nextStart.getTime() + latency * 1500));
                    LOG.debug("Adding to pending jobs: " + key + " ---> " + SchemaHelper.formatDateUTC(nextStart));
                    Calendar startCal = Calendar.getInstance(timeZone);
                    startCal.setTime(nextStart);
                    startCal.add(frequency.getTimeUnit().getCalendarUnit(), frequency.getFrequencyAsInt());
                    nextStart = startCal.getTime();
                }
            }
        }
View Full Code Here

        throws FalconException {
        CONTROLS controls = new CONTROLS();
        controls.setConcurrency(String.valueOf(process.getParallel()));
        controls.setExecution(process.getOrder().name());

        Frequency timeout = process.getTimeout();
        long frequencyInMillis = ExpressionHelper.get().evaluate(process.getFrequency().toString(), Long.class);
        long timeoutInMillis;
        if (timeout != null) {
            timeoutInMillis = ExpressionHelper.get().
                    evaluate(process.getTimeout().toString(), Long.class);
View Full Code Here

                return null;
            }
            Retry retry = new Retry();
            retry.setAttempts(Integer.parseInt(RuntimeProperties.get()
                    .getProperty("feed.retry.attempts", "3")));
            retry.setDelay(new Frequency(RuntimeProperties.get().getProperty(
                    "feed.retry.frequency", "minutes(5)")));
            retry.setPolicy(PolicyType.fromValue(RuntimeProperties.get()
                    .getProperty("feed.retry.policy", "exp-backoff")));
            return retry;
        case PROCESS:
View Full Code Here

                    .getProperty("feed.late.allowed", "true")
                    .equalsIgnoreCase("true")) {
                return null;
            }
            LateProcess lateProcess = new LateProcess();
            lateProcess.setDelay(new Frequency(RuntimeProperties.get()
                    .getProperty("feed.late.frequency", "hours(3)")));
            lateProcess.setPolicy(PolicyType.fromValue(RuntimeProperties.get()
                    .getProperty("feed.late.policy", "exp-backoff")));
            LateInput lateInput = new LateInput();
            lateInput.setInput(entity.getName());
View Full Code Here

            replicationCoord.getControls().setTimeout(String.valueOf(timeoutInMillis / (1000 * 60)));
            replicationCoord.getControls().setThrottle(String.valueOf(timeoutInMillis / frequencyInMillis * 2));
            replicationCoord.getControls().setConcurrency(String.valueOf(parallel));

            Frequency replicationDelay = FeedHelper.getCluster(feed,
                    srcCluster.getName()).getDelay();
            long delayInMillis=0;
            if (replicationDelay != null) {
                delayInMillis = ExpressionHelper.get().evaluate(
                        replicationDelay.toString(), Long.class);
                long delayInMins = -1 * delayInMillis / (1000 * 60);
                String elExp = "${now(0," + delayInMins + ")}";
                replicationCoord.getInputEvents().getDataIn().get(0)
                .getInstance().set(0, elExp);
                replicationCoord.getOutputEvents().getDataOut().get(0)
View Full Code Here

TOP

Related Classes of org.apache.falcon.entity.v0.Frequency

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.