Examples of Storage


Examples of org.apache.falcon.entity.Storage

        String storageType = props.get("falconFeedStorageType");

        // verify the param that feed evictor depends on
        Assert.assertEquals(storageType, Storage.TYPE.FILESYSTEM.name());

        final Storage storage = FeedHelper.createStorage(cluster, feed);
        if (feedDataPath != null) {
            Assert.assertEquals(feedDataPath, storage.getUriTemplate()
                    .replaceAll(Storage.DOLLAR_EXPR_START_REGEX, Storage.QUESTION_EXPR_START_REGEX));
        }

        if (storageType != null) {
            Assert.assertEquals(storageType, storage.getType().name());
        }

        // verify the post processing params
        Assert.assertEquals(props.get("feedNames"), feed.getName());
        Assert.assertEquals(props.get("feedInstancePaths"), "IGNORE");
View Full Code Here

Examples of org.apache.falcon.entity.Storage

        List<String> inputFeeds = new ArrayList<String>();
        List<String> inputPaths = new ArrayList<String>();
        List<String> inputFeedStorageTypes = new ArrayList<String>();
        for (Input input : process.getInputs().getInputs()) {
            Feed feed = EntityUtil.getEntity(EntityType.FEED, input.getFeed());
            Storage storage = FeedHelper.createStorage(cluster, feed);

            if (!input.isOptional()) {
                if (coord.getDatasets() == null) {
                    coord.setDatasets(new DATASETS());
                }
                if (coord.getInputEvents() == null) {
                    coord.setInputEvents(new INPUTEVENTS());
                }

                SYNCDATASET syncdataset = createDataSet(feed, cluster, storage, input.getName(), LocationType.DATA);
                coord.getDatasets().getDatasetOrAsyncDataset().add(syncdataset);

                DATAIN datain = createDataIn(input);
                coord.getInputEvents().getDataIn().add(datain);
            }

            String inputExpr = null;
            if (storage.getType() == Storage.TYPE.FILESYSTEM) {
                inputExpr = getELExpression("dataIn('" + input.getName() + "', '" + input.getPartition() + "')");
                props.put(input.getName(), inputExpr);
            } else if (storage.getType() == Storage.TYPE.TABLE) {
                inputExpr = "${coord:dataIn('" + input.getName() + "')}";
                propagateCatalogTableProperties(input, (CatalogStorage) storage, props);
            }

            inputFeeds.add(input.getName());
            inputPaths.add(inputExpr);
            inputFeedStorageTypes.add(storage.getType().name());
        }

        propagateLateDataProperties(inputFeeds, inputPaths, inputFeedStorageTypes, props);
    }
View Full Code Here

Examples of org.apache.falcon.entity.Storage

        List<String> outputFeeds = new ArrayList<String>();
        List<String> outputPaths = new ArrayList<String>();
        for (Output output : process.getOutputs().getOutputs()) {
            Feed feed = EntityUtil.getEntity(EntityType.FEED, output.getFeed());
            Storage storage = FeedHelper.createStorage(cluster, feed);

            SYNCDATASET syncdataset = createDataSet(feed, cluster, storage, output.getName(), LocationType.DATA);
            coord.getDatasets().getDatasetOrAsyncDataset().add(syncdataset);

            DATAOUT dataout = createDataOut(output);
            coord.getOutputEvents().getDataOut().add(dataout);

            String outputExpr = "${coord:dataOut('" + output.getName() + "')}";
            outputFeeds.add(output.getName());
            outputPaths.add(outputExpr);

            if (storage.getType() == Storage.TYPE.FILESYSTEM) {
                props.put(output.getName(), outputExpr);

                propagateFileSystemProperties(output, feed, cluster, coord, storage, props);
            } else if (storage.getType() == Storage.TYPE.TABLE) {
                propagateCatalogTableProperties(output, (CatalogStorage) storage, props);
            }
        }

        // Output feed name and path for parent workflow
View Full Code Here

Examples of org.apache.falcon.entity.Storage

    private void addInputFeedsAsParams(List<String> paramList, Process process, Cluster cluster,
                                       String engineType) throws FalconException {
        for (Input input : process.getInputs().getInputs()) {
            Feed feed = EntityUtil.getEntity(EntityType.FEED, input.getFeed());
            Storage storage = FeedHelper.createStorage(cluster, feed);

            final String inputName = input.getName();
            if (storage.getType() == Storage.TYPE.FILESYSTEM) {
                paramList.add(inputName + "=${" + inputName + "}"); // no prefix for backwards compatibility
            } else if (storage.getType() == Storage.TYPE.TABLE) {
                final String paramName = "falcon_" + inputName; // prefix 'falcon' for new params
                Map<String, String> props = new HashMap<String, String>();
                propagateCommonCatalogTableProperties((CatalogStorage) storage, props, paramName);
                for (String key : props.keySet()) {
                    paramList.add(key + "=${wf:conf('" + key + "')}");
View Full Code Here

Examples of org.apache.falcon.entity.Storage

    private void addOutputFeedsAsParams(List<String> paramList, Process process,
                                        Cluster cluster) throws FalconException {
        for (Output output : process.getOutputs().getOutputs()) {
            Feed feed = EntityUtil.getEntity(EntityType.FEED, output.getFeed());
            Storage storage = FeedHelper.createStorage(cluster, feed);

            if (storage.getType() == Storage.TYPE.FILESYSTEM) {
                final String outputName = output.getName()// no prefix for backwards compatibility
                paramList.add(outputName + "=${" + outputName + "}");
            } else if (storage.getType() == Storage.TYPE.TABLE) {
                Map<String, String> props = new HashMap<String, String>();
                propagateCatalogTableProperties(output, (CatalogStorage) storage, props); // prefix is auto added
                for (String key : props.keySet()) {
                    paramList.add(key + "=${wf:conf('" + key + "')}");
                }
View Full Code Here

Examples of org.apache.falcon.entity.Storage

     * @param retention retention limit
     * @throws FalconException
     */
    private void deleteStagedData(Cluster cluster, Feed feed, long retention)
        throws FalconException {
        Storage storage = FeedHelper.createStorage(cluster, feed);
        if (storage.getType() == Storage.TYPE.FILESYSTEM) {  // FS does NOT use staging dirs
            return;
        }

        final CatalogStorage tableStorage = (CatalogStorage) storage;
        String stagingDir = FeedHelper.getStagingDir(cluster, feed, tableStorage, Tag.REPLICATION);
View Full Code Here

Examples of org.apache.falcon.entity.Storage

        }
    }

    public static boolean shouldUpdate(Feed oldFeed, Feed newFeed, Process affectedProcess)
        throws FalconException {
        Storage oldFeedStorage = FeedHelper.createStorage(oldFeed);
        Storage newFeedStorage = FeedHelper.createStorage(newFeed);

        if (!oldFeedStorage.isIdentical(newFeedStorage)) {
            return true;
        }
        LOG.debug(oldFeed.toShortString() + ": Storage identical. Ignoring...");
View Full Code Here

Examples of org.apache.falcon.entity.Storage

        LOG.info("Applying retention on " + feedBasePath + " type: " + retentionType
                + ", Limit: " + retentionLimit + ", timezone: " + timeZone
                + ", frequency: " + frequency + ", storage" + feedStorageType);

        Storage storage = FeedHelper.createStorage(feedStorageType, feedBasePath);
        evict(storage, retentionLimit, timeZone);

        logInstancePaths(new Path(logFile));

        int len = buffer.length();
View Full Code Here

Examples of org.apache.falcon.entity.Storage

            Map<String, String> props = createCoordDefaultConfiguration(cluster, wfPath, wfName);
            props.put("timeZone", feed.getTimezone().getID());
            props.put("frequency", feed.getFrequency().getTimeUnit().name());

            final Storage storage = FeedHelper.createStorage(cluster, feed);
            props.put("falconFeedStorageType", storage.getType().name());

            String feedDataPath = storage.getUriTemplate();
            props.put("feedDataPath",
                    feedDataPath.replaceAll(Storage.DOLLAR_EXPR_START_REGEX, Storage.QUESTION_EXPR_START_REGEX));

            org.apache.falcon.entity.v0.feed.Cluster feedCluster =
                    FeedHelper.getCluster(feed, cluster.getName());
View Full Code Here

Examples of org.apache.falcon.entity.Storage

                    ? SchemaHelper.formatDateUTC(sourceEndDate) : SchemaHelper.formatDateUTC(targetEndDate);

            initializeCoordAttributes(replicationCoord, coordName, feed, start, end, replicationDelayInMillis);
            setCoordControls(feed, replicationCoord);

            final Storage sourceStorage = FeedHelper.createReadOnlyStorage(srcCluster, feed);
            initializeInputDataSet(feed, srcCluster, replicationCoord, sourceStorage);

            final Storage targetStorage = FeedHelper.createStorage(trgCluster, feed);
            initializeOutputDataSet(feed, trgCluster, replicationCoord, targetStorage);

            Path wfPath = getCoordPath(bundlePath, coordName);
            ACTION replicationWorkflowAction = getReplicationWorkflowAction(
                    srcCluster, trgCluster, wfPath, coordName, sourceStorage, targetStorage);
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.