Package org.apache.falcon.entity.v0.process

Examples of org.apache.falcon.entity.v0.process.Process


        resource = this.getClass().getResource("/config/feed/hive-table-feed-out.xml");
        Feed outFeed = (Feed) EntityType.FEED.getUnmarshaller().unmarshal(resource);
        ConfigurationStore.get().publish(EntityType.FEED, outFeed);

        resource = this.getClass().getResource("/config/process/hive-process.xml");
        Process process = (Process) EntityType.PROCESS.getUnmarshaller().unmarshal(resource);
        ConfigurationStore.get().publish(EntityType.PROCESS, process);

        prepare(process);
        OozieProcessWorkflowBuilder builder = new OozieProcessWorkflowBuilder(process);
        Path bundlePath = new Path("/falcon/staging/workflows", process.getName());
        builder.map(cluster, bundlePath);
        assertTrue(fs.exists(bundlePath));

        BUNDLEAPP bundle = getBundle(bundlePath);
        assertEquals(EntityUtil.getWorkflowName(process).toString(), bundle.getName());
        assertEquals(1, bundle.getCoordinator().size());
        assertEquals(EntityUtil.getWorkflowName(Tag.DEFAULT, process).toString(),
                bundle.getCoordinator().get(0).getName());
        String coordPath = bundle.getCoordinator().get(0).getAppPath().replace("${nameNode}", "");

        COORDINATORAPP coord = getCoordinator(new Path(coordPath));
        HashMap<String, String> props = new HashMap<String, String>();
        for (Property prop : coord.getAction().getWorkflow().getConfiguration().getProperty()) {
            props.put(prop.getName(), prop.getValue());
        }

        // verify table props
        Map<String, String> expected = getExpectedProperties(inFeed, outFeed, process);
        for (Map.Entry<String, String> entry : props.entrySet()) {
            if (expected.containsKey(entry.getKey())) {
                Assert.assertEquals(entry.getValue(), expected.get(entry.getKey()));
            }
        }

        String wfPath = coord.getAction().getWorkflow().getAppPath().replace("${nameNode}", "");
        WORKFLOWAPP parentWorkflow = getParentWorkflow(new Path(wfPath));
        testParentWorkflow(process, parentWorkflow);

        List<Object> decisionOrForkOrJoin = parentWorkflow.getDecisionOrForkOrJoin();

        ACTION hiveNode = (ACTION) decisionOrForkOrJoin.get(4);
        Assert.assertEquals("user-hive-job", hiveNode.getName());

        JAXBElement<org.apache.falcon.oozie.hive.ACTION> actionJaxbElement = OozieUtils.unMarshalHiveAction(hiveNode);
        org.apache.falcon.oozie.hive.ACTION hiveAction = actionJaxbElement.getValue();

        Assert.assertEquals(hiveAction.getScript(),
                "${nameNode}/falcon/staging/workflows/hive-process/user/script.hql");
        Assert.assertEquals(hiveAction.getJobXml(), "${wf:appPath()}/conf/hive-site.xml");
        Assert.assertNull(hiveAction.getPrepare());
        Assert.assertEquals(Collections.EMPTY_LIST, hiveAction.getArchive());
        Assert.assertFalse(hiveAction.getParam().isEmpty());
        Assert.assertEquals(11, hiveAction.getParam().size());

        Assert.assertTrue(Storage.TYPE.TABLE == ProcessHelper.getStorageType(cluster, process));
        assertHCatCredentials(parentWorkflow, wfPath);

        ConfigurationStore.get().remove(EntityType.PROCESS, process.getName());
    }
View Full Code Here


        resource = this.getClass().getResource("/config/feed/hive-table-feed-out.xml");
        Feed outFeed = (Feed) EntityType.FEED.getUnmarshaller().unmarshal(resource);
        ConfigurationStore.get().publish(EntityType.FEED, outFeed);

        resource = this.getClass().getResource("/config/process/pig-process-table.xml");
        Process process = (Process) EntityType.PROCESS.getUnmarshaller().unmarshal(resource);
        ConfigurationStore.get().publish(EntityType.PROCESS, process);

        OozieProcessWorkflowBuilder builder = new OozieProcessWorkflowBuilder(process);
        Path bundlePath = new Path("/falcon/staging/workflows", process.getName());
        builder.map(cluster, bundlePath);
        assertTrue(fs.exists(bundlePath));

        BUNDLEAPP bundle = getBundle(bundlePath);
        assertEquals(EntityUtil.getWorkflowName(process).toString(), bundle.getName());
        assertEquals(1, bundle.getCoordinator().size());
        assertEquals(EntityUtil.getWorkflowName(Tag.DEFAULT, process).toString(),
                bundle.getCoordinator().get(0).getName());
        String coordPath = bundle.getCoordinator().get(0).getAppPath().replace("${nameNode}", "");

        COORDINATORAPP coord = getCoordinator(new Path(coordPath));
        HashMap<String, String> props = new HashMap<String, String>();
        for (Property prop : coord.getAction().getWorkflow().getConfiguration().getProperty()) {
            props.put(prop.getName(), prop.getValue());
        }

        // verify table props
        Map<String, String> expected = getExpectedProperties(inFeed, outFeed, process);
        for (Map.Entry<String, String> entry : props.entrySet()) {
            if (expected.containsKey(entry.getKey())) {
                Assert.assertEquals(entry.getValue(), expected.get(entry.getKey()));
            }
        }

        // verify the late data params
        Assert.assertEquals(props.get("falconInputFeeds"), process.getInputs().getInputs().get(0).getFeed());
        Assert.assertEquals(props.get("falconInPaths"), "${coord:dataIn('input')}");
        Assert.assertEquals(props.get("falconInputFeedStorageTypes"), Storage.TYPE.TABLE.name());

        // verify the post processing params
        Assert.assertEquals(props.get("feedNames"), process.getOutputs().getOutputs().get(0).getFeed());
        Assert.assertEquals(props.get("feedInstancePaths"), "${coord:dataOut('output')}");

        String wfPath = coord.getAction().getWorkflow().getAppPath().replace("${nameNode}", "");
        WORKFLOWAPP parentWorkflow = getParentWorkflow(new Path(wfPath));
View Full Code Here

        }
    }

    @Test
    public void testProcessWorkflowMapper() throws Exception {
        Process process = ConfigurationStore.get().get(EntityType.PROCESS, "clicksummary");
        Workflow processWorkflow = process.getWorkflow();
        Assert.assertEquals("test", processWorkflow.getName());
        Assert.assertEquals("1.0.0", processWorkflow.getVersion());
    }
View Full Code Here

    @Test
    public void testProcessWithNoInputsAndOutputs() throws Exception {
        ClusterHelper.getInterface(cluster, Interfacetype.WRITE).setEndpoint(hdfsUrl);

        URL resource = this.getClass().getResource("/config/process/dumb-process.xml");
        Process processEntity = (Process) EntityType.PROCESS.getUnmarshaller().unmarshal(resource);
        ConfigurationStore.get().publish(EntityType.PROCESS, processEntity);

        OozieProcessWorkflowBuilder builder = new OozieProcessWorkflowBuilder(processEntity);
        Path bundlePath = new Path("/falcon/staging/workflows", processEntity.getName());
        builder.map(cluster, bundlePath);
        assertTrue(fs.exists(bundlePath));

        BUNDLEAPP bundle = getBundle(bundlePath);
        assertEquals(EntityUtil.getWorkflowName(processEntity).toString(), bundle.getName());
View Full Code Here

                }
            }
            return (T) feed;

        case PROCESS:
            Process process = (Process) entity.copy();
            Iterator<org.apache.falcon.entity.v0.process.Cluster> procItr =
                process.getClusters().getClusters().iterator();
            while (procItr.hasNext()) {
                org.apache.falcon.entity.v0.process.Cluster cluster = procItr.next();
                if (!cluster.getName().equals(clusterName)) {
                    procItr.remove();
                }
View Full Code Here

                clusters.add(cluster.getName());
            }
            break;

        case PROCESS:
            Process process = (Process) entity;
            for (org.apache.falcon.entity.v0.process.Cluster cluster : process.getClusters().getClusters()) {
                clusters.add(cluster.getName());
            }
            break;
        default:
        }
View Full Code Here

                    "feed.retry.frequency", "minutes(5)")));
            retry.setPolicy(PolicyType.fromValue(RuntimeProperties.get()
                    .getProperty("feed.retry.policy", "exp-backoff")));
            return retry;
        case PROCESS:
            Process process = (Process) entity;
            return process.getRetry();
        default:
            throw new FalconException("Cannot create Retry for entity:" + entity.getName());
        }
    }
View Full Code Here

            //TODO - Assuming the late workflow is not used
            lateInput.setWorkflowPath("ignore.xml");
            lateProcess.getLateInputs().add(lateInput);
            return lateProcess;
        case PROCESS:
            Process process = (Process) entity;
            return process.getLateProcess();
        default:
            throw new FalconException("Cannot create Late Process for entity:" + entity.getName());
        }
    }
View Full Code Here

        }
    }

    public static Process buildProcess(String processName, Cluster cluster,
                                       String tags) throws Exception {
        Process processEntity = new Process();
        processEntity.setName(processName);
        processEntity.setTags(tags);

        org.apache.falcon.entity.v0.process.Cluster processCluster =
                new org.apache.falcon.entity.v0.process.Cluster();
        processCluster.setName(cluster.getName());
        processEntity.setClusters(new Clusters());
        processEntity.getClusters().getClusters().add(processCluster);

        return processEntity;
    }
View Full Code Here

        output.setFeed(feed.getName());
        outputs.getOutputs().add(output);
    }

    public void addProcessEntity() throws Exception {
        Process processEntity = buildProcess(PROCESS_ENTITY_NAME, clusterEntity, "classified-as=Critical");
        addWorkflow(processEntity, WORKFLOW_NAME, WORKFLOW_VERSION);

        for (Feed inputFeed : inputFeeds) {
            addInput(processEntity, inputFeed);
        }
View Full Code Here

TOP

Related Classes of org.apache.falcon.entity.v0.process.Process

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.