Package org.apache.falcon.entity.v0

Examples of org.apache.falcon.entity.v0.Frequency

404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
                        .getResourceAsStream("/config/feed/feed-0.1.xml"));
        feed2.setName("feed2");
        feed2.getLocations().getLocations().get(0).setPath("/data/clicks/${YEAR}/${MONTH}/${DAY}/${HOUR}");
        feed2.getClusters().getClusters().get(0).getLocations().getLocations()
                .get(0).setPath("/data/clicks/${YEAR}/${MONTH}/${DAY}/${HOUR}");
        feed2.setFrequency(new Frequency("hours(1)"));
        try {
            parser.parseAndValidate(feed2.toString());
        } catch (FalconException e) {
            e.printStackTrace();
            Assert.fail("Not expecting exception for same frequency");
        }
        feed2.setFrequency(new Frequency("hours(2)"));
        //expecting exception
        parser.parseAndValidate(feed2.toString());
    }
View Full Code Here
81
82
83
84
85
86
87
88
89
90
91
        storeEntity(EntityType.FEED, "imp-click-join2");
        storeEntity(EntityType.PROCESS, "sample");
        Process process = ConfigurationStore.get().get(EntityType.PROCESS, "sample");
        Process otherProcess = (Process) process.copy();
        otherProcess.setName("sample2");
        otherProcess.setFrequency(new Frequency("days(1)"));
        ConfigurationStore.get().remove(EntityType.PROCESS,
                otherProcess.getName());
        ConfigurationStore.get().publish(EntityType.PROCESS, otherProcess);

        fs.mkdirs(instanceLogPath);
View Full Code Here
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
                if (entity == null) {
                    LOG.warn("No entity for " + key);
                    continue;
                }
                Date startTime = EntityUtil.getStartTime(entity, cluster);
                Frequency frequency = EntityUtil.getFrequency(entity);
                TimeZone timeZone = EntityUtil.getTimeZone(entity);
                Date nextStart = EntityUtil.getNextStartTime(startTime, frequency, timeZone, now);
                if (nextStart.after(windowEndTime)) {
                    continue;
                }
                ConcurrentMap<Date, Date> pendingInstances = pendingJobs.get(key);
                while (!nextStart.after(windowEndTime)) {
                    if (pendingInstances == null) {
                        pendingJobs.putIfAbsent(key, new ConcurrentHashMap<Date, Date>());
                        pendingInstances = pendingJobs.get(key);
                    }
                    Long latency = monitoredEntities.get(key);
                    if (latency == null) {
                        break;
                    }
                    // 1.5 times latency is when it is supposed to have breached
                    pendingInstances.putIfAbsent(nextStart, new Date(nextStart.getTime() + latency * 1500));
                    LOG.debug("Adding to pending jobs: " + key + " ---> " + SchemaHelper.formatDateUTC(nextStart));
                    Calendar startCal = Calendar.getInstance(timeZone);
                    startCal.setTime(nextStart);
                    startCal.add(frequency.getTimeUnit().getCalendarUnit(), frequency.getFrequency());
                    nextStart = startCal.getTime();
                }
            }
        }
View Full Code Here
87
88
89
90
91
92
93
94
95
96
97
98
99
    public void testClusterEntityWithInvalidInterfaces(Interfacetype interfacetype, String endpoint)
        throws Exception {
        overlay = context.getUniqueOverlay();
        String filePath = context.overlayParametersOverTemplate(TestContext.CLUSTER_TEMPLATE, overlay);
        InputStream stream = new FileInputStream(filePath);
        Cluster cluster = (Cluster) EntityType.CLUSTER.getUnmarshaller().unmarshal(stream);
        Assert.assertNotNull(cluster);
        cluster.setColo("default");  // validations will be ignored if not default & tests fail

        Interface anInterface = ClusterHelper.getInterface(cluster, interfacetype);
        anInterface.setEndpoint(endpoint);

        File tmpFile = context.getTempFile();
View Full Code Here
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
    private static final Logger LOG = Logger.getLogger(OozieWorkflowBuilder.class);
    protected static final ConfigurationStore CONFIG_STORE = ConfigurationStore.get();

    protected Properties createAppProperties(String clusterName, Path bundlePath, String user) throws FalconException {

        Cluster cluster = EntityUtil.getEntity(EntityType.CLUSTER, clusterName);
        Properties properties = new Properties();
        if (cluster.getProperties() != null) {
            addClusterProperties(properties, cluster.getProperties()
                    .getProperties());
        }
        properties.setProperty(OozieWorkflowEngine.NAME_NODE,
                ClusterHelper.getStorageUrl(cluster));
        properties.setProperty(OozieWorkflowEngine.JOB_TRACKER,
                ClusterHelper.getMREndPoint(cluster));
        properties.setProperty(OozieClient.BUNDLE_APP_PATH,
                "${" + OozieWorkflowEngine.NAME_NODE + "}" + bundlePath.toString());
        properties.setProperty("colo.name", cluster.getColo());

        properties.setProperty(OozieClient.USER_NAME, user);
        properties.setProperty(OozieClient.USE_SYSTEM_LIBPATH, "true");
        properties.setProperty("falcon.libpath", ClusterHelper.getLocation(cluster, "working") + "/lib");
        LOG.info("Cluster: " + cluster.getName() + ", PROPS: " + properties);
        return properties;
    }
View Full Code Here
61
62
63
64
65
66
67
68
69
70
71
        org.apache.falcon.entity.v0.feed.Cluster feedCluster = FeedHelper.getCluster(feed, clusterName);
        if (!startDate.before(feedCluster.getValidity().getEnd())) {
            return null;
        }

        Cluster cluster = CONFIG_STORE.get(EntityType.CLUSTER, feedCluster.getName());
        Path bundlePath = new Path(ClusterHelper.getLocation(cluster, "staging"), EntityUtil.getStagingPath(feed));
        Feed feedClone = (Feed) feed.copy();
        EntityUtil.setStartDate(feedClone, clusterName, startDate);

        AbstractOozieEntityMapper<Feed> mapper = new OozieFeedMapper(feedClone);
View Full Code Here
106
107
108
109
110
111
112
113
114
115
116
        Entity entity = (Entity) unmarshaller
                .unmarshal(OozieFeedMapperTest.class.getResource(template));
        store.publish(type, entity);

        if (type == EntityType.CLUSTER) {
            Cluster cluster = (Cluster) entity;
            ClusterHelper.getInterface(cluster, Interfacetype.WRITE).setEndpoint(writeEndpoint);
            FileSystem fs = new Path(writeEndpoint).getFileSystem(new Configuration());
            fs.create(new Path(ClusterHelper.getLocation(cluster, "working"), "libext/FEED/retention/ext.jar")).close();
            fs.create(
                    new Path(ClusterHelper.getLocation(cluster, "working"), "libext/FEED/replication/ext.jar")).close();
View Full Code Here
54
55
56
57
58
59
60
61
62
    public void onAdd(Entity entity, boolean ignoreFailure) {
        if (entity.getEntityType() != EntityType.CLUSTER) {
            return;
        }

        Cluster cluster = (Cluster) entity;
        COLO_CLUSTER_MAP.putIfAbsent(cluster.getColo(), new HashSet<String>());
        COLO_CLUSTER_MAP.get(cluster.getColo()).add(cluster.getName());
    }
View Full Code Here
65
66
67
68
69
70
71
72
73
74
75
    public void onRemove(Entity entity) {
        if (entity.getEntityType() != EntityType.CLUSTER) {
            return;
        }

        Cluster cluster = (Cluster) entity;
        COLO_CLUSTER_MAP.get(cluster.getColo()).remove(cluster.getName());
        if (COLO_CLUSTER_MAP.get(cluster.getColo()).isEmpty()) {
            COLO_CLUSTER_MAP.remove(cluster.getColo());
        }
    }
View Full Code Here
30
31
32
33
34
35
36
37
38
39
*/
@Test
public class ColoClusterRelationTest extends AbstractTestBase {

    private Cluster newCluster(String name, String colo) {
        Cluster cluster = new Cluster();
        cluster.setName(name);
        cluster.setColo(colo);
        return cluster;
    }
View Full Code Here
TOP

Related Classes of org.apache.falcon.entity.v0.Frequency

  • com.ibm.icu.util.Calendar
  • org.apache.falcon.cleanup.FeedCleanupHandler
  • org.apache.falcon.cleanup.LogCleanupServiceTest
  • org.apache.falcon.cleanup.ProcessCleanupHandler
  • org.apache.falcon.cluster.util.EmbeddedCluster
  • org.apache.falcon.converter.AbstractTestBase
  • org.apache.falcon.converter.OozieFeedMapper
  • org.apache.falcon.converter.OozieFeedMapper$ReplicationOozieWorkflowMapper
  • org.apache.falcon.converter.OozieFeedMapper$RetentionOozieWorkflowMapper
  • org.apache.falcon.converter.OozieFeedMapperTest
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.