Package org.apache.falcon.entity.v0

Examples of org.apache.falcon.entity.v0.Frequency


  }

  @Test
  public void TestPeriodicPolicy() throws IvoryException, InterruptedException {
    AbstractRerunPolicy periodic = new PeriodicPolicy();
    long delay = periodic.getDelay(new Frequency("minutes(2)"), 2);
    Assert.assertEquals(delay, 120000);
    delay = periodic.getDelay(new Frequency("minutes(2)"), 5);
    Assert.assertEquals(delay, 120000);

    long currentTime = System.currentTimeMillis();
    //Thread.sleep(1000);
    delay = periodic.getDelay(new Frequency("minutes(3)"), new Date(
        currentTime), new Date(currentTime + 1 * 60
        * 60 * 1000));
    Assert.assertEquals(delay, 180000);
  }
View Full Code Here


     
            replicationCoord.getControls().setTimeout(String.valueOf(timeout_ms / (1000 * 60)));
            replicationCoord.getControls().setThrottle(String.valueOf(timeout_ms / frequency_ms * 2));
            replicationCoord.getControls().setConcurrency(String.valueOf(parallel));
           
      Frequency replicationDelay = FeedHelper.getCluster(feed,
          srcCluster.getName()).getDelay();
      long delay_ms=0;
      if (replicationDelay != null) {
        delay_ms = ExpressionHelper.get().evaluate(
            replicationDelay.toString(), Long.class);
        long delay_mins = -1 * delay_ms / (1000 * 60);
        String elExp = "${now(0," + delay_mins + ")}";
        replicationCoord.getInputEvents().getDataIn().get(0)
            .getInstance().set(0, elExp);
        replicationCoord.getOutputEvents().getDataOut().get(0)
View Full Code Here

                if (entity == null) {
                    LOG.warn("No entity for " + key);
                    continue;
                }
                Date startTime = EntityUtil.getStartTime(entity, cluster);
                Frequency frequency = EntityUtil.getFrequency(entity);
                TimeZone timeZone = EntityUtil.getTimeZone(entity);
                Date nextStart = EntityUtil.getNextStartTime(startTime, frequency, timeZone, now);
                if (nextStart.after(windowEndTime)) continue;
                ConcurrentMap<Date, Date> pendingInstances = pendingJobs.get(key);
                while (!nextStart.after(windowEndTime)) {
                    if (pendingInstances == null) {
                        pendingJobs.putIfAbsent(key, new ConcurrentHashMap<Date, Date>());
                        pendingInstances = pendingJobs.get(key);
                    }
                    Long latency = monitoredEntities.get(key);
                    if (latency == null) break;
                    pendingInstances.putIfAbsent(nextStart, new Date(nextStart.getTime() +
                            latency * 1500))//1.5 times latency is when it is supposed to have breached
                    LOG.debug("Adding to pending jobs: " + key + " ---> " +
                            SchemaHelper.formatDateUTC(nextStart));
                    Calendar startCal = Calendar.getInstance(timeZone);
                    startCal.setTime(nextStart);
                    startCal.add(frequency.getTimeUnit().getCalendarUnit(), frequency.getFrequency());
                    nextStart = startCal.getTime();
                }
            }
        }
View Full Code Here

    storeEntity(EntityType.PROCESS, "sample");
    Process process = ConfigurationStore.get().get(EntityType.PROCESS,
        "sample");
    Process otherProcess = (Process) process.clone();
    otherProcess.setName("sample2");
    otherProcess.setFrequency(new Frequency("days(1)"));
    ConfigurationStore.get().remove(EntityType.PROCESS,
        otherProcess.getName());
    ConfigurationStore.get().publish(EntityType.PROCESS, otherProcess);   

    fs.mkdirs(instanceLogPath);
View Full Code Here

        return null;
      }
      Retry retry = new Retry();
      retry.setAttempts(Integer.parseInt(RuntimeProperties.get()
          .getProperty("feed.retry.attempts", "3")));
      retry.setDelay(new Frequency(RuntimeProperties.get().getProperty(
          "feed.retry.frequency", "minutes(5)")));
      retry.setPolicy(PolicyType.fromValue(RuntimeProperties.get()
          .getProperty("feed.retry.policy", "exp-backoff")));
      return retry;
    case PROCESS:
View Full Code Here

          .getProperty("feed.late.allowed", "true")
          .equalsIgnoreCase("true")) {
        return null;
      }
      LateProcess lateProcess = new LateProcess();
      lateProcess.setDelay(new Frequency(RuntimeProperties.get()
          .getProperty("feed.late.frequency", "hours(3)")));
      lateProcess.setPolicy(PolicyType.fromValue(RuntimeProperties.get()
          .getProperty("feed.late.policy", "exp-backoff")));
      LateInput lateInput = new LateInput();
      lateInput.setInput(entity.getName());
View Full Code Here

            + entity.getName() + "), ignoring failed retries");
        return;
      }

      int attempts = retry.getAttempts();
      Frequency delay = retry.getDelay();
      PolicyType policy = retry.getPolicy();
      int intRunId = Integer.parseInt(runId);

      if (attempts > intRunId) {
        AbstractRerunPolicy rerunPolicy = RerunPolicyFactory
View Full Code Here

  @Override
  public void run() {
    int attempt = 1;
    AbstractRerunPolicy policy = new ExpBackoffPolicy();
    Frequency frequency = new Frequency("minutes(1)");
    while (true) {
      try {
        T message = null;
        try {
          message = handler.takeFromQueue();
View Full Code Here

        // controls
        CONTROLS controls = new CONTROLS();
        controls.setConcurrency(String.valueOf(process.getParallel()));
        controls.setExecution(process.getOrder().name());

        Frequency timeout = process.getTimeout();
        long frequency_ms = ExpressionHelper.get().
                evaluate(process.getFrequency().toString(), Long.class);
        long timeout_ms;
        if (timeout != null) {
            timeout_ms = ExpressionHelper.get().
View Full Code Here

            .getResourceAsStream("/config/feed/feed-0.1.xml"));
    feed2.setName("feed2");
    feed2.getLocations().getLocations().get(0).setPath("/data/clicks/${YEAR}/${MONTH}/${DAY}/${HOUR}");
    feed2.getClusters().getClusters().get(0).getLocations().getLocations()
    .get(0).setPath("/data/clicks/${YEAR}/${MONTH}/${DAY}/${HOUR}");
    feed2.setFrequency(new Frequency("hours(1)"));
    try{
      parser.parseAndValidate(feed2.toString());
    }catch(IvoryException e){
      e.printStackTrace();
      Assert.fail("Not expecting exception for same frequency");
    }
    feed2.setFrequency(new Frequency("hours(2)"));
    //expecting exception
    parser.parseAndValidate(feed2.toString());
  }
View Full Code Here

TOP

Related Classes of org.apache.falcon.entity.v0.Frequency

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.