Package cascading.flow.hadoop

Examples of cascading.flow.hadoop.HadoopFlowConnector

55
56
57
58
59
60
61
        Tap in = sourceTap();
        Tap out = new EsTap("cascading-hadoop/artists", new Fields("name", "url", "picture"));
        Pipe pipe = new Pipe("copy");

        FlowDef flowDef = FlowDef.flowDef().addSource(pipe, in).addTailSink(pipe, out);
        StatsUtils.proxy(new HadoopFlowConnector(HdpBootstrap.asProperties(CascadingHadoopSuite.configuration)).connect(flowDef)).complete();
    }
View Full Code Here
74
75
76
77
78
79
80
        // rename "id" -> "garbage"
        pipe = new Each(pipe, new Identity(new Fields("garbage", "name", "url", "picture", "ts")));

        Properties props = HdpBootstrap.asProperties(CascadingHadoopSuite.configuration);
        props.setProperty("es.mapping.names", "url:address");
        StatsUtils.proxy(new HadoopFlowConnector(props).connect(in, out, pipe)).complete();
    }
View Full Code Here
89
90
91
92
93
94
95
        Properties props = HdpBootstrap.asProperties(CascadingHadoopSuite.configuration);

        Tap in = sourceTap();
        Tap out = new EsTap("cascading-hadoop/pattern-{id}", new Fields("id", "name", "url", "picture"));
        Pipe pipe = new Pipe("copy");
        StatsUtils.proxy(new HadoopFlowConnector(props).connect(in, out, pipe)).complete();
    }
View Full Code Here
104
105
106
107
108
109
110
        Properties props = HdpBootstrap.asProperties(CascadingHadoopSuite.configuration);

        Tap in = sourceTap();
        Tap out = new EsTap("cascading-hadoop/pattern-format-{ts:YYYY-MM-dd}", new Fields("id", "name", "url", "picture", "ts"));
        Pipe pipe = new Pipe("copy");
        StatsUtils.proxy(new HadoopFlowConnector(props).connect(in, out, pipe)).complete();
    }
View Full Code Here
121
122
123
124
125
126
127
128
129
130
131
        Properties cfg = HdpBootstrap.asProperties(CascadingHadoopSuite.configuration);

        FlowDef flow = new FlowDef().addSource(copy, sourceTap())
                .addTailSink(copy, new EsTap("cascading-hadoop/cascade-connector"));

        FlowConnector connector = new HadoopFlowConnector(cfg);
        Flow[] flows = new Flow[] { connector.connect(flow) };

        CascadeConnector cascadeConnector = new CascadeConnector(cfg);
        cascadeConnector.connect(flows).complete();
    }
View Full Code Here
75
76
77
78
79
80
81
        Pipe pipe = new Pipe("copy");
        build(cfg(), in, out, pipe);
    }

    private void build(Properties cfg, Tap in, Tap out, Pipe pipe) {
        StatsUtils.proxy(new HadoopFlowConnector(cfg).connect(in, out, pipe)).complete();
    }
View Full Code Here
122
123
124
125
126
127
128
        Assert.assertTrue(RestUtils.exists(indexPrefix + "cascading-hadoop/pattern-format-2198-10-06"));
        Assert.assertTrue(RestUtils.exists(indexPrefix + "cascading-hadoop/pattern-format-2890-10-06"));
    }

    private void build(Properties cfg, Tap in, Tap out, Pipe pipe) {
        StatsUtils.proxy(new HadoopFlowConnector(cfg).connect(in, out, pipe)).complete();
    }
View Full Code Here
159
160
161
162
163
164
165
        return props;
    }

    private void build(Properties cfg, Tap in, Tap out, Pipe pipe) {
        StatsUtils.proxy(new HadoopFlowConnector(cfg).connect(in, out, pipe)).complete();
    }
View Full Code Here
96
97
98
99
100
101
102
103
104
105
        FlowDef flowDef = new FlowDef()
            .setName("group-DRMs-by-key")
            .addSource(lhs, dRM1Source)
            .addSource(rhs, dRM2Source)
            .addTailSink(groupByItemIDPipe, groupedOutputSink);
        Flow flow = new HadoopFlowConnector().connect(flowDef);
        flow.complete();

        //todo: not sure if it matters but may need to rename the part files to .csv
    }
View Full Code Here
119
120
121
122
123
124
125
126
127
128
        FlowDef flowDef = new FlowDef()
            .setName("convert-to-CSV")
            .addSource(dRM1, dRM1Source)
            .addTailSink(dRM1, outputSink);
        Flow flow = new HadoopFlowConnector().connect(flowDef);
        flow.complete();

        //todo: not sure if it matters but may need to rename the part files to .csv
    }
View Full Code Here
TOP

Related Classes of cascading.flow.hadoop.HadoopFlowConnector

  • cascading.flow.hadoop.planner.HadoopPlanner
  • cascading.flow.hadoop.planner.MapReduceHadoopRuleRegistry
  • cascading.lingual.platform.hadoop.HadoopPlatformBroker
  • cascading.pattern.Main
  • cascading.platform.hadoop.HadoopPlatform
  • com.backtype.support.CascadingUtils
  • com.liveramp.cascading_ext.example.BloomJoinExampleWithoutCascadingUtil
  • com.scaleunlimited.cascading.scheme.hadoop.SolrSchemeHadoopTest
  • com.squareup.cascading2.function.ExtractProtoTest
  • com.squareup.cascading2.function.TestExpandProto
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.