Examples of Storage


Examples of org.apache.falcon.entity.Storage

        return processes;
    }

    private void validateFeedGroups(Feed feed) throws FalconException {
        String[] groupNames = feed.getGroups() != null ? feed.getGroups().split(",") : new String[]{};
        final Storage storage = FeedHelper.createStorage(feed);
        String defaultPath = storage.getUriTemplate(LocationType.DATA);
        for (Cluster cluster : feed.getClusters().getClusters()) {
            final String uriTemplate = FeedHelper.createStorage(cluster, feed).getUriTemplate(LocationType.DATA);
            if (!FeedGroup.getDatePattern(uriTemplate).equals(
                    FeedGroup.getDatePattern(defaultPath))) {
                throw new ValidationException("Feeds default path pattern: "
                        + storage.getUriTemplate(LocationType.DATA)
                        + ", does not match with cluster: "
                        + cluster.getName()
                        + " path pattern: "
                        + uriTemplate);
            }
View Full Code Here

Examples of org.apache.falcon.entity.Storage

                    EntityUtil.getEntity(EntityType.CLUSTER, cluster.getName());
            if (!EntityUtil.responsibleFor(clusterEntity.getColo())) {
                continue;
            }

            final Storage storage = FeedHelper.createStorage(cluster, feed);
            if (!storage.exists()) {
                // this is only true for table, filesystem always returns true
                CatalogStorage catalogStorage = (CatalogStorage) storage;
                throw new ValidationException("Table [" + catalogStorage.getTable()
                        + "] does not exist for feed: " + feed.getName() + ", cluster: " + cluster.getName());
            }
View Full Code Here

Examples of org.apache.falcon.entity.Storage

        return processes;
    }

    private void validateFeedGroups(Feed feed) throws FalconException {
        String[] groupNames = feed.getGroups() != null ? feed.getGroups().split(",") : new String[]{};
        final Storage storage = FeedHelper.createStorage(feed);
        String defaultPath = storage.getUriTemplate(LocationType.DATA);
        for (Cluster cluster : feed.getClusters().getClusters()) {
            final String uriTemplate = FeedHelper.createStorage(cluster, feed).getUriTemplate(LocationType.DATA);
            if (!FeedGroup.getDatePattern(uriTemplate).equals(
                    FeedGroup.getDatePattern(defaultPath))) {
                throw new ValidationException("Feeds default path pattern: "
                        + storage.getUriTemplate(LocationType.DATA)
                        + ", does not match with cluster: "
                        + cluster.getName()
                        + " path pattern: "
                        + uriTemplate);
            }
View Full Code Here

Examples of org.apache.falcon.entity.Storage

        String storageType = props.get("falconFeedStorageType");

        // verify the param that feed evictor depends on
        Assert.assertEquals(storageType, Storage.TYPE.FILESYSTEM.name());

        final Storage storage = FeedHelper.createStorage(cluster, feed);
        if (feedDataPath != null) {
            Assert.assertEquals(feedDataPath, storage.getUriTemplate()
                    .replaceAll(Storage.DOLLAR_EXPR_START_REGEX, Storage.QUESTION_EXPR_START_REGEX));
        }

        if (storageType != null) {
            Assert.assertEquals(storageType, storage.getType().name());
        }

        // verify the post processing params
        Assert.assertEquals(props.get("feedNames"), feed.getName());
        Assert.assertEquals(props.get("feedInstancePaths"), "IGNORE");
View Full Code Here

Examples of org.apache.falcon.entity.Storage

        String storageType = props.get("falconFeedStorageType");

        // verify the param that feed evictor depends on
        Assert.assertEquals(storageType, Storage.TYPE.TABLE.name());

        final Storage storage = FeedHelper.createStorage(cluster, tableFeed);
        if (feedDataPath != null) {
            Assert.assertEquals(feedDataPath, storage.getUriTemplate()
                    .replaceAll(Storage.DOLLAR_EXPR_START_REGEX, Storage.QUESTION_EXPR_START_REGEX));
        }

        if (storageType != null) {
            Assert.assertEquals(storageType, storage.getType().name());
        }

        // verify the post processing params
        Assert.assertEquals(props.get("feedNames"), tableFeed.getName());
        Assert.assertEquals(props.get("feedInstancePaths"), "IGNORE");
View Full Code Here

Examples of org.apache.falcon.entity.Storage

            Map<String, String> props = createCoordDefaultConfiguration(cluster, wfPath, wfName);
            props.put("timeZone", entity.getTimezone().getID());
            props.put("frequency", entity.getFrequency().getTimeUnit().name());

            final Storage storage = FeedHelper.createStorage(cluster, entity);
            props.put("falconFeedStorageType", storage.getType().name());

            String feedDataPath = storage.getUriTemplate();
            props.put("feedDataPath",
                feedDataPath.replaceAll(Storage.DOLLAR_EXPR_START_REGEX, Storage.QUESTION_EXPR_START_REGEX));

            org.apache.falcon.entity.v0.feed.Cluster feedCluster = FeedHelper.getCluster(entity, cluster.getName());
            props.put("limit", feedCluster.getRetention().getLimit().toString());
View Full Code Here

Examples of org.apache.falcon.entity.Storage

                ? SchemaHelper.formatDateUTC(sourceEndDate) : SchemaHelper.formatDateUTC(targetEndDate);

            initializeCoordAttributes(replicationCoord, coordName, feed, start, end, replicationDelayInMillis);
            setCoordControls(feed, replicationCoord);

            final Storage sourceStorage = FeedHelper.createReadOnlyStorage(srcCluster, feed);
            initializeInputDataSet(feed, srcCluster, replicationCoord, sourceStorage);

            final Storage targetStorage = FeedHelper.createStorage(trgCluster, feed);
            initializeOutputDataSet(feed, trgCluster, replicationCoord, targetStorage);

            ACTION replicationWorkflowAction = getReplicationWorkflowAction(
                srcCluster, trgCluster, wfPath, coordName, sourceStorage, targetStorage);
            replicationCoord.setAction(replicationWorkflowAction);
View Full Code Here

Examples of org.apache.falcon.entity.Storage

        List<String> inputFeeds = new ArrayList<String>();
        List<String> inputPaths = new ArrayList<String>();
        List<String> inputFeedStorageTypes = new ArrayList<String>();
        for (Input input : process.getInputs().getInputs()) {
            Feed feed = EntityUtil.getEntity(EntityType.FEED, input.getFeed());
            Storage storage = FeedHelper.createStorage(cluster, feed);

            if (!input.isOptional()) {
                if (coord.getDatasets() == null) {
                    coord.setDatasets(new DATASETS());
                }
                if (coord.getInputEvents() == null) {
                    coord.setInputEvents(new INPUTEVENTS());
                }

                SYNCDATASET syncdataset = createDataSet(feed, cluster, storage, input.getName(), LocationType.DATA);
                coord.getDatasets().getDatasetOrAsyncDataset().add(syncdataset);

                DATAIN datain = createDataIn(input);
                coord.getInputEvents().getDataIn().add(datain);
            }

            String inputExpr = null;
            if (storage.getType() == Storage.TYPE.FILESYSTEM) {
                inputExpr = getELExpression("dataIn('" + input.getName() + "', '" + input.getPartition() + "')");
                props.put(input.getName(), inputExpr);
            } else if (storage.getType() == Storage.TYPE.TABLE) {
                inputExpr = "${coord:dataIn('" + input.getName() + "')}";
                propagateCatalogTableProperties(input, (CatalogStorage) storage, props);
            }

            inputFeeds.add(feed.getName());
            inputPaths.add(inputExpr);
            inputFeedStorageTypes.add(storage.getType().name());
        }

        propagateLateDataProperties(inputFeeds, inputPaths, inputFeedStorageTypes, props);
    }
View Full Code Here

Examples of org.apache.falcon.entity.Storage

        List<String> outputFeeds = new ArrayList<String>();
        List<String> outputPaths = new ArrayList<String>();
        for (Output output : process.getOutputs().getOutputs()) {
            Feed feed = EntityUtil.getEntity(EntityType.FEED, output.getFeed());
            Storage storage = FeedHelper.createStorage(cluster, feed);

            SYNCDATASET syncdataset = createDataSet(feed, cluster, storage, output.getName(), LocationType.DATA);
            coord.getDatasets().getDatasetOrAsyncDataset().add(syncdataset);

            DATAOUT dataout = createDataOut(output);
            coord.getOutputEvents().getDataOut().add(dataout);

            String outputExpr = "${coord:dataOut('" + output.getName() + "')}";
            outputFeeds.add(feed.getName());
            outputPaths.add(outputExpr);

            if (storage.getType() == Storage.TYPE.FILESYSTEM) {
                props.put(output.getName(), outputExpr);

                propagateFileSystemProperties(output, feed, cluster, coord, storage, props);
            } else if (storage.getType() == Storage.TYPE.TABLE) {
                propagateCatalogTableProperties(output, (CatalogStorage) storage, props);
            }
        }

        // Output feed name and path for parent workflow
View Full Code Here

Examples of org.apache.falcon.entity.Storage

            return;
        }

        for (Input input : process.getInputs().getInputs()) {
            Feed feed = EntityUtil.getEntity(EntityType.FEED, input.getFeed());
            Storage storage = FeedHelper.createStorage(cluster, feed);

            final String inputName = input.getName();
            if (storage.getType() == Storage.TYPE.FILESYSTEM) {
                paramList.add(inputName + "=${" + inputName + "}"); // no prefix for backwards compatibility
            } else if (storage.getType() == Storage.TYPE.TABLE) {
                final String paramName = "falcon_" + inputName; // prefix 'falcon' for new params
                Map<String, String> props = new HashMap<String, String>();
                propagateCommonCatalogTableProperties((CatalogStorage) storage, props, paramName);
                for (String key : props.keySet()) {
                    paramList.add(key + "=${wf:conf('" + key + "')}");
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.